results visualization in bokeh (but still need to automatically generate results),

partial integration of tgview

interview_kernel/exaoutput.py

@@ -6,6 +6,7 @@ from pylatexenc.latexencode import utf8tolatex, utf82latex
 
 
 def remove_ensuremaths():
+    """remove ensuremath wrappers when translating back from unicode to latex"""
     thisdict = utf82latex
     for key, value in utf82latex.items():
         if value.startswith('\\ensuremath{'):
@@ -15,66 +16,60 @@ def remove_ensuremaths():
 
 
 class ExaOutput:
-    def __init__(self):
+    """generates configuration files for exastencils,
+        but only if simdata is given"""
+    def __init__(self, simdata=None, username="user", probname=None):
         remove_ensuremaths()
+        self.exastencils_path = Path("./exastencils")
 
-    def create_output(self, simdata):
-        self.filespathpre = "Configs"  # just to build paths, remove
-        self.probname = "Poisson_1D"  # just to build paths, remove
+        self.username = username
+
+        if probname is None:
+            self.probname = self.username
         #output parameters which should also be made adaptable at some point
-#        self.knowledge = OrderedDict([
-#                           ("dimensionality" , simdata["num_dimensions"]),
-#                           ("minLevel" , 1),
-#                           ("maxLevel" , 7),
-#                           ("discr_type" , simdata["sim_type"]),
-#                           ("l3tmp_generateL4" , False),
-#                           ("experimental_layerExtension" , True)
-#                           ])
         self.platform = OrderedDict([
-                            ("targetOS" , "Linux"),
-                           ("targetCompiler" , "GCC"),
-                           ("targetCompilerVersion" , 5),
-                           ("targetCompilerVersionMinor" , 4),
-                           ("simd_instructionSet" , "AVX")
+                            ("targetOS", "Linux"),
+                           ("targetCompiler", "GCC"),
+                           ("targetCompilerVersion", 5),
+                           ("targetCompilerVersionMinor", 4),
+                           ("simd_instructionSet", "AVX")
                            ])
         self.settings = OrderedDict([
-            ("user", "Guest"),
-            ("configName", "1D_FD_Poisson_fromL1"),
-            ("basePathPrefix", "./Poisson"),
-            ("l1file", "\"$configName$.exa1\""),
-            ("debugL1File", "\"../Debug/$configName$_debug.exa1\""),
-            ("debugL2File", "\"../Debug/$configName$_debug.exa2\""),
-            ("debugL3File", "\"../Debug/$configName$_debug.exa3\""),
-            ("debugL4File", "\"../Debug/$configName$_debug.exa4\""),
+            ("user", self.username),
+            ("configName", self.probname),
+            ("basePathPrefix", "./" + self.probname),
+            ("l1file", "$configName$.exa1"),
+            ("debugL1File", "../Debug/$configName$_debug.exa1"),
+            ("debugL2File", "../Debug/$configName$_debug.exa2"),
+            ("debugL3File", "../Debug/$configName$_debug.exa3"),
+            ("debugL4File", "../Debug/$configName$_debug.exa4"),
             ("htmlLogFile", "../Debug/$configName$_log.html"),
             ("outputPath", "../generated/$configName$/"),
-            ("produceHtmlLog", "true"),
-            ("timeStrategies", "true"),
+            ("produceHtmlLog", True),
+            ("timeStrategies", True),
             ("buildfileGenerators", "{ \"MakefileGenerator\" }")
         ])
-        filespath = Path(self.filespathpre).joinpath(self.settings["user"]).joinpath(self.probname)
-        self.l1file = filespath.with_suffix('.exa1')
-        self.l2file = filespath.with_suffix('.exa2')
-        self.l3file = filespath.with_suffix('.exa3')
-        self.l4file = filespath.with_suffix('.exa4')
-        ff = str(Path(self.settings["basePathPrefix"]).joinpath(self.filespathpre).joinpath(self.settings["user"]))
+        self.dirpath = self.exastencils_path.joinpath(self.probname)
+        self.filespath = self.dirpath.joinpath(self.probname)
+        ff = str(self.dirpath)
         if not os.path.exists(ff):
             try:
                 os.makedirs(ff)
             except OSError as exc:# Guard against race condition
                 if exc.errno != errno.EEXIST:
                     raise
-
-        self.create_settings()
-#        self.create_platform()
-        self.create_knowledge()
-        self.create_l1(simdata)
-#        self.create_l2(simdata)
-#        self.create_l3()
-#        self.create_l4()
+        if simdata is not None:
+            self.create_settings()
+    #        self.create_platform()
+            self.create_knowledge()
+            self.create_l1(simdata)
+            self.create_examples_list_file()
+    #        self.create_l2(simdata)
+    #        self.create_l3()
+    #        self.create_l4()
 
     def create_l1(self, simdata):
-        l1path = str(Path(self.settings["basePathPrefix"]).joinpath(self.l1file))
+        l1path = str(self.filespath.with_suffix('.exa1'))
         domain_name = utf8tolatex(simdata["domain"]["name"], non_ascii_only=True, brackets=False)
         op = utf8tolatex(simdata["pdes"]["pdes"][-1]["op"], non_ascii_only=True, brackets=False)
         bcrhs = self.replace_boundary_x(simdata["bcs"]["bcs"][-1]["rhsstring_expanded"]) #TODO expand
@@ -128,6 +123,7 @@ class ExaOutput:
                 "ApplicationHints { // alt L4Hint(s) \n"
                 "  // parameters \n"
                 "  l4_genDefaultApplication = true \n"
+                "  l4_defAppl_FieldToPrint = \"u\" \n" #TODO
                 "} \n"
            )
 
@@ -138,7 +134,7 @@ class ExaOutput:
         return string.replace("x", "vf_boundaryCoord_x")
 
     def create_l2(self, simdata):
-        l2path = str(Path(self.settings["basePathPrefix"]).joinpath(self.l2file))
+        l2path = str(self.filespath.with_suffix('.exa2'))
         with open(l2path,'w') as l2:
             l2.write(
                 "Domain global< " + simdata["domain"]["from"] + " to " + simdata["domain"]["to"] + " > \n" # TODO domain goes here
@@ -164,14 +160,14 @@ class ExaOutput:
             )
 
     def create_l3(self):
-        l3path = str(Path(self.settings["basePathPrefix"]).joinpath(self.l3file))
-        with open(l3path,'w') as l3:
+        l3path = str(self.filespath.with_suffix('.exa3'))
+        with open(l3path, 'w') as l3:
             l3.write(
                 "generate solver for Solution in solEq \n"
             )
 
     def create_l4(self):
-        l4path = str(Path(self.settings["basePathPrefix"]).joinpath(self.l4file))
+        l4path = str(self.filespath.with_suffix('.exa4'))
         with open(l4path,'w') as l4:
             l4.write(
                 "Function Application ( ) : Unit { \n"
@@ -201,25 +197,31 @@ class ExaOutput:
         return key.ljust(30) + ' = ' + val + '\n'
 
     def format_key_val(self, key, val):
-        return self.key_val(key, repr(val).replace('\'', '\"'))
+        val_repr = repr(val).replace('\'', '\"')
+        if isinstance(val, bool):
+            val_repr = val_repr.lower()
+        return self.key_val(key, val_repr)
 
     def format_key(self, key, dict):
         return self.format_key_val(key, dict[key])
 
     def create_settings(self):
-        settingspath = str(Path(self.settings["basePathPrefix"]).joinpath('settings'))
-        with open(settingspath,'w') as settingsfile:
+        settingspath = str(self.filespath) + '.settings'
+        with open(settingspath, 'w') as settingsfile:
             for key in self.settings:
-                settingsfile.write(self.format_key(key, self.settings))
+                if key == "buildfileGenerators":
+                    settingsfile.write(self.key_val(key, self.settings[key]))
+                else:
+                    settingsfile.write(self.format_key(key, self.settings))
 
     def create_platform(self):
-        platformpath = str(Path(self.settings["basePathPrefix"]).joinpath('platform'))
+        platformpath = str(self.filespath) + '.platform'
         with open(platformpath,'w') as platformfile:
             for key in self.platform:
                 platformfile.write(self.format_key(key, self.platform))
 
     def create_knowledge(self):
-        knowledgepath = str(Path(self.settings["basePathPrefix"]).joinpath('knowledge'))
+        knowledgepath = str(self.filespath) + '.knowledge'
         with open(knowledgepath,'w') as knowledgefile:
 #            for key in self.knowledge:
 #                knowledgefile.write(self.format_key(key, self.knowledge))
@@ -228,3 +230,30 @@ class ExaOutput:
                 "import '../lib/domain_onePatch.knowledge' \n"
                 "import '../lib/parallelization_pureOmp.knowledge'"
             )
+
+    def create_examples_list_file(self):
+        examples_path = str(self.exastencils_path.joinpath("examples").with_suffix('.sh'))
+        with open(examples_path, 'w') as shell_file:
+            shell_file.write(
+                "#!/usr/bin/env bash \n"\
+                "\n"
+                "\n"
+                "configList=\"\" \n"\
+                "configList+=\"{}/{} \" \n".format(self.probname, self.probname))
+
+class ExaRunner():
+    from functools import lru_cache
+
+    def __init__(self, exaout):
+        self.exaout = exaout
+
+    @lru_cache()
+    def load_data(self, data_name="u"):
+        import pandas as pd
+        data_path = self.exaout.exastencils_path.joinpath("generated").joinpath(self.exaout.probname).joinpath(data_name).with_suffix(".dat")
+        df = pd.read_csv(data_path, sep=' ', index_col=0)
+        try:
+            df.columns = ['u']
+        except ValueError:  # length mismatch because additional column of nans was read
+            df.columns = ['u', 'nan']
+        return df

interview_kernel/install.py

@@ -3,14 +3,17 @@ import json
 import os
 import sys
 
+from interview_kernel import Interview
+
 from jupyter_client.kernelspec import KernelSpecManager
 from IPython.utils.tempdir import TemporaryDirectory
 
-kernel_json = {
-    "argv": [sys.executable, "-m", "interview_kernel", "-f", "{connection_file}"],
-    "display_name": "Interview",
-    "language": "text",
-}
+kernel_json = Interview.kernel_json
+#{
+#    "argv": [sys.executable, "-m", "interview_kernel", "-f", "{connection_file}"],
+#    "display_name": "Interview",
+#    "language": "text",
+#}
 
 def install_my_kernel_spec(user=True, prefix=None):
     with TemporaryDirectory() as td:

interview_kernel/interview.py

@@ -10,8 +10,14 @@ import pyparsing as pp
 
 from pde_state_machine import *
 
-# This "main class" is two things: a REPL loop, by subclassing the cmd2 Cmd class
-# and a state machine as given by the pytransitions package
+import matplotlib
+matplotlib.use('nbagg')
+import matplotlib.pyplot as plt
+from bokeh.io import output_notebook, show, export_svgs
+from bokeh.plotting import figure
+from tempfile import NamedTemporaryFile
+
+# This "main class" is a REPL loop, by subclassing the cmd2 Cmd class
 class Interview(cmd.Cmd):
     def __init__(self, *args, **kwargs):
 
@@ -155,6 +161,73 @@ class Interview(cmd.Cmd):
                                                                   "Let's set up a simulation together.\n")
         self.trigger("greeting_over")
 
+    def do_plt(self, userstring):
+        plt.ion()
+
+        plot = plt.plot([3, 8, 2, 5, 1])
+        #self.Display(plot)
+        plt.show() #TODO find out why there is no comm and interactive shell - and if it should be there
+
+        # cf. http://ipython-books.github.io/16-creating-a-simple-kernel-for-jupyter/
+        fig = plt.figure()
+        ax = fig.add_subplot(111)
+        ax.plot([3, 8, 2, 5, 1])
+
+        # We create a PNG out of this plot.
+        png = _to_png(fig)
+
+        with open('png.png', 'w') as f:
+            f.write(png)
+
+        # and send it along as rich content
+        self.richcontent = dict()
+
+        # We prepare the response with our rich data (the plot).
+        self.richcontent['source'] = 'kernel'
+
+        # This dictionary may contain different MIME representations of the output.
+        self.richcontent['data'] = {
+                                       'image/png': png  # TODO error: Notebook JSON is invalid: [{'image/png': ...
+                                   },
+        # We can specify the image size in the metadata field.
+        self.richcontent['metadata'] = {
+            'image/png': {
+                'width': 600,
+                'height': 400
+            }
+        }
+        self.poutput("image!")
+
+    # cf. nbviewer.jupyter.org/github/bokeh/bokeh-notebooks/blob/master/tutorial/01 - Basic Plotting.ipynb
+    def do_bokeh(self, userstring):
+        # create a new plot with default tools, using figure
+        p = figure(plot_width=400, plot_height=400)
+
+        # add a circle renderer with a size, color, and alpha
+        p.circle([1, 2, 3, 4, 5], [6, 7, 2, 4, 5], size=15, line_color="navy", fill_color="orange", fill_alpha=0.5)
+        show(p)
+        t = NamedTemporaryFile(suffix="svg")
+        p.output_backend = "svg"
+        export_svgs(p, filename=t.name)
+        svg = open(t.name).buffer
+
+        # and send it along as rich content
+        self.richcontent = dict()
+
+        # We prepare the response with our rich data (the plot).
+        self.richcontent['source'] = 'kernel'
+
+        # This dictionary may contain different MIME representations of the output.
+        self.richcontent['data'] = {
+                                       'image/svg': svg
+                                   },
+        # We can specify the image size in the metadata field.
+        self.richcontent['metadata'] = {
+            'image/svg': {
+                'width': 600,
+                'height': 400
+            }
+        }
 
 if __name__ == '__main__':
     Interview().cmdloop()

interview_kernel/interview_kernel.py

+from sys import executable
+from os.path import join
+#from pathlib import Path
 from metakernel import MetaKernel
 from IPython.display import HTML, Javascript
 from metakernel import IPythonKernel
@@ -6,10 +9,17 @@ from metakernel import IPythonKernel
 # https://github.com/phfaist/pylatexenc for directly converting Latex commands to unicode
 from pylatexenc.latex2text import LatexNodes2Text
 import getpass
-from pde_state_machine import *
+import matplotlib
+matplotlib.use('nbagg')
 import matplotlib.pyplot as plt
+from bokeh.io import output_notebook, show, export_svgs
+from bokeh.plotting import figure
+from bokeh.resources import CDN
+from bokeh.embed import file_html, components#, notebook_div
+from bokeh.models import ColumnDataSource
+#from tempfile import NamedTemporaryFile
 
-
+from pde_state_machine import *
 
 # This "main class" is two things: a REPL loop, by subclassing the cmd2 Cmd class
 # and a state machine as given by the pytransitions package
@@ -25,17 +35,18 @@ class Interview(MetaKernel):
         'help_links': MetaKernel.help_links,
     }
     banner = "Interview kernel\n\n" \
-             "Hello, " + getpass.getuser() + "! I am " + "TheInterview" + ", your partial differential equations and simulations expert. " \
-                                                                           "Let's set up a simulation together.\n" \
+             "Hello, " + getpass.getuser() + "! I am " + "TheInterview" + \
+             ", your partial differential equations and simulations expert. " \
+             "Let's set up a simulation together.\n" \
              "Please enter anything to start the interview."
 
-    #kernel_json = {
-    #    "argv": [
-    #        sys.executable, "-m", "interview_kernel", "-f", "{connection_file}"],
-    #    "display_name": "Interview Kernel",
-    #    "language": "text",
-    #    "name": "interview_kernel"
-    #}
+    kernel_json = {
+        "argv": [
+            executable, "-m", "interview_kernel", "-f", "{connection_file}"],
+        "display_name": "TheInterview",
+        "language": "text",
+        "name": "interview_kernel"
+    }
 
     def __init__(self, **kwargs):
 
@@ -45,7 +56,7 @@ class Interview(MetaKernel):
         super(Interview, self).__init__(**kwargs)
 
         self.do_execute("%matplotlib nbagg")
-        plt.ion()
+        #plt.ion()
 
         # To make custom magics happen, cf. https://github.com/Calysto/metakernel
         # from IPython import get_ipython
@@ -55,6 +66,10 @@ class Interview(MetaKernel):
         self.update_prompt()
         self.poutstring = ""# to collect string output to send
         self.outstream_name = 'stdout'
+        self.richcontent = None  # to collect rich contents (images etc)
+
+        # bokeh notebook setup
+        output_notebook()
 
     def poutput(self, text, outstream_name='stdout'):
         """Accumulate the output here"""
@@ -71,26 +86,23 @@ class Interview(MetaKernel):
 
         if not self.keyword_handling(arg):
             if not self.prompt_input_handling(arg):
-                self.state_input_handling(arg)
+                self.state_machine.handle_state_dependent_input(arg)
 
         if not silent:
             stream_content = {'name': self.outstream_name, 'text': self.poutstring}
             self.send_response(self.iopub_socket, 'stream', stream_content)
 
+        if self.richcontent is not None:
+            # We send the display_data message with the contents.
+            self.send_response(self.iopub_socket, 'display_data', self.richcontent)
+
+            self.richcontent = None
+
         self.poutstring = ""
         self.outstream_name = 'stdout'
 
         return  # stream_content['text']
 
-    def state_input_handling(self, arg):
-        """The standard input handling, depending on which state we are in"""
-        # pythonic switch-case, cf. https://bytebaker.com/2008/11/03/switch-case-statement-in-python/
-        try:
-            self.state_machine.stateDependentInputHandling[self.state_machine.state](arg)
-        except Exception as error:
-            #self.state_machine.exaout.create_output(self.simdata)
-            raise
-
     def please_prompt(self, query, if_yes, if_no=None, pass_other=False):
         self.poutput(str(query) + " [y/n]? ")
         self.state_machine.prompted = True
@@ -137,6 +149,9 @@ class Interview(MetaKernel):
         if arg.startswith("plt"):
             self.display_plt()
             return True
+        if arg.startswith("bokeh"):
+            self.display_bokeh()
+            return True
         return False
 
     # called when user types 'explain [expression]'
@@ -172,31 +187,25 @@ class Interview(MetaKernel):
         return super(Interview, self).do_shutdown(restart)
 
     # tab completion for empty lines
-    def completenames(self, text, line, begidx, endidx):
+    def do_complete(self, code, cursor_pos):
         """Override of cmd2 method which completes command names both for command completion and help."""
-        command = text
-        if self.case_insensitive:
-            command = text.lower()
-        if not command:
-            # define the "default" input for the different states we can be in
-            self.stateDependentDefaultInput = {
-                'dimensions': '1',
-                'domain': ['Ω = [ 0 ; 1 ]'],
-                'unknowns': ['u : Ω → ℝ'],
-                'parameters': ['f :  ℝ → ℝ = [x: ℝ] x '],  # ['f : Ω → ℝ = [x:Ω] x ⋅ x'],
-                'pdes': ['∆u = f(x_1)'],
-                'bcs': ['u (0) = 0'],  # ,'u (1) = x_1**2'],
-                'sim': ['FD'],
-            }
-            return self.stateDependentDefaultInput[self.state_machine.state]
+        # define the "default" input for the different states we can be in
+        state_dependent_default_input = {
+            'greeting': 'hi',
+            'dimensions': '1',
+            'domain': ['Ω = [ 0 ; 1 ]'],
+            'unknowns': ['u : Ω → ℝ'],
+            'parameters': ['f :  ℝ → ℝ = [x: ℝ] x '],  # ['f : Ω → ℝ = [x:Ω] x ⋅ x'],
+            'pdes': ['∆u = f(x)'],
+            'bcs': ['u = 0'],  # ,'u (1) = x_1**2'],
+            'sim': ['FD'],
+        }
+        if not code or state_dependent_default_input[self.state_machine.state].startswith(code):
+            return state_dependent_default_input[self.state_machine.state]
         else:
-            # Call super class method.  Need to do it this way for Python 2 and 3 compatibility
-            cmd_completion = cmd.Cmd.completenames(self, command)
-
-            # If we are completing the initial command name and get exactly 1 result and are at end of line, add a space
-            if begidx == 0 and len(cmd_completion) == 1 and endidx == len(line):
-                cmd_completion[0] += ' '
-            return cmd_completion
+            # Call super class method.
+            super(Interview, self).do_complete(code, cursor_pos)
+            return
 
     def display_html(self):
         self.Display(HTML("""
@@ -217,18 +226,85 @@ class Interview(MetaKernel):
         </script>
                             """ % (1, 0, 3, 0)))
 
+        tgview_url = join(self.state_machine.mmtinterface.serverInstance, "graphs/tgview.html?type=thgraph&graphdata=", self.state_machine.mmtinterface.URIprefix, self.state_machine.mmtinterface.namespace + "?u")
+
         othercode = """
-            <iframe>
-            ### have tgview here
+            <iframe 
+                src="https://mmt.mathhub.info/graphs/tgview.html" 
+                style="width: 100%; height: 510px; border: none"
+            >
             </iframe>
         """
+        # a futile attempt to set the size
+        metadata = {
+                        #'text/html': {
+                            'width': 1600,
+                            'height': 1400
+                        #}
+                    }
+        self.Display(HTML(othercode, metadata=metadata))
+
+    # cf. nbviewer.jupyter.org/github/bokeh/bokeh-notebooks/blob/master/tutorial/01 - Basic Plotting.ipynb
+    def display_bokeh(self):
+        from exaoutput import ExaOutput, ExaRunner
+
+        # create a new plot with default tools, using figure
+        p = figure(plot_width=1000, plot_height=400)
+
+        runner = ExaRunner(ExaOutput())
+        data = runner.load_data("u")
+        #source = ColumnDataSource(data=data)
+        source = ColumnDataSource(data=dict(x=[], u=[]))
+        source.data = source.from_df(data )#[['x', 'u']])
+        source.add(data.index, 'index')
+
+        # add a circle renderer with a size, color, and alpha
+        p.circle(x='index', y='u', size=2, line_color="navy", fill_color="orange", fill_alpha=0.5, source=source)
+        #show(p)
+
+        output_notebook()
+        # cf. http://bokeh.pydata.org/en/0.10.0/docs/user_guide/embed.html
+        self.Display(HTML(file_html(p, CDN, "my plot")))  # show the results
+
+        # using JS requires jupyter widgets extension
+        #script, div = components(p)
+        #div = notebook_div(p)
+        #self.Display(Javascript(script + div))  # show the results
+
 
     def display_plt(self):
         # plt.ion()
-        # matplotlib.use('nbagg')
-        self.Display(plt.plot([3, 8, 2, 5, 1]))
+
+        # plot = plt.plot([3, 8, 2, 5, 1])
+        #self.Display(plot)
         # plt.show() #TODO find out why there is no comm and interactive shell - and if it should be there
 
+        # cf. http://ipython-books.github.io/16-creating-a-simple-kernel-for-jupyter/
+        fig = plt.figure()
+        ax = fig.add_subplot(111)
+        ax.plot([3, 8, 2, 5, 1])
+
+        # We create a PNG out of this plot.
+        png = _to_png(fig)
+
+        # and send it along as rich content
+        self.richcontent = dict()
+
+        # We prepare the response with our rich data (the plot).
+        self.richcontent['source'] = 'kernel'
+
+        # This dictionary may contain different MIME representations of the output.
+        self.richcontent['data'] = {
+                                        'image/png': png  #TODO error: Notebook JSON is invalid: [{'image/png': ...
+                                    },
+        # We can specify the image size in the metadata field.
+        self.richcontent['metadata'] = {
+                                            'image/png': {
+                                                'width': 600,
+                                                'height': 400
+                                            }
+                                        }
+        self.poutput("image!")
 
 if __name__ == '__main__':
     # from ipykernel.kernelapp import IPKernelApp

interview_kernel/mmtinterface.py

@@ -191,6 +191,12 @@ class MMTInterface:
         self.namespace = 'MitM/smglom/calculus'  # TODO
         self.debugprint = False
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.exit_mmt()
+
     def exit_mmt(self):
         exit_mmt_server(self.port_number, self.mmt_jar)
 

interview_kernel/pde_state_machine.py

@@ -112,8 +112,6 @@ class PDE_States:
             'sim': self.sim_handle_input,
         }
 </