fixing pipeline

.gitlab-ci.yml

@@ -7,22 +7,22 @@ test_snipper:
   stage: test
   script:
     - echo "Testing student files"
-#    - pwd
     - apt-get update -y
     - apt install -y python3-pip
     - apt install -y python3.10 python-is-python3
+    - apt install -y git
+    - apt install -y xvfb # Virtual framebuffer for GL stuff.
+    - pip install -e ./
+    - cd tests
+    - xvfb-run -s "-screen 0 1400x900x24" python -m unittest test_python.py
+    - echo "Buy world"
+
+
+
 #    - apt install -y python-is-python3
 #    - python --version
 #    - locate python
-    - apt install -y git
-    - apt install -y xvfb # Virtual framebuffer for GL stuff.
 #    - apt install -y inkscape pdftk
 #    - apt install -y latexmk  poppler-utils # latexmk and pdftocairo
 #    - DEBIAN_FRONTEND='noninteractive' apt install -y texlive-latex-recommended texlive-latex-extra texlive-science pdf2svg
 #    - pip install -U Pillow
-    - pip install -e ./
-    - cd tests
-    - xvfb-run -s "-screen 0 1400x900x24" python -m unittest test_python.py
-#    - latexmk --version
-#    - ls tests_images/*
-

src/codesnipper.egg-info/PKG-INFO

 Metadata-Version: 2.1
 Name: codesnipper
-Version: 0.1.14
+Version: 0.1.17
 Summary: A lightweight framework for censoring student solutions files and extracting code + output
 Home-page: https://lab.compute.dtu.dk/tuhe/snipper
 Author: Tue Herlau

src/snipper/fix_bf.py

@@ -66,7 +66,7 @@ def fix_f(lines, debug, keep=False):
     return lines2
 
 # stats = {'n': 0}
-def _block_fun(lines, start_extra, end_extra, keep=False, permute=False, questionmarks=False, halfquestionmarks=False, silent=False):
+def _block_fun(lines, start_extra, end_extra, keep=False, permute=False, questionmarks=False, halfquestionmarks=False, silent=False, nolines=False):
     methods = {'remove': 0}
     # if method not in ['remove', 'permute', 'questionmark', 'halfquestionmark']:
     #     assert False
@@ -105,7 +105,12 @@ def _block_fun(lines, start_extra, end_extra, keep=False, permute=False, questio
                 lines = f3(lines)
                 insert_lines = True
             if not insert_lines:
-                lines = [id + f"# TODO: {cc} lines missing."]
+                if nolines:
+                    todo = f"# TODO: Code has been removed from here."
+                else:
+                    todo = f"# TODO: {cc} lines missing."
+
+                lines = [id + todo]
             else:
                 lines = msg + lines
             lines += [id + f'raise NotImplementedError("{ee}")']

src/snipper/fix_cite.py

@@ -9,8 +9,8 @@ def fix_citations(lines, references, strict=True, file=None):
     I.e. rst files should get rst references.
 
     """
-    if str(file).endswith(".rst"):
-        print(file)
+    # if str(file).endswith(".rst"):
+    #     print(file)
     # lines = fix_aux(lines, )
     lines = fix_single_reference(lines, aux=references.get('aux', {}), cmd="\\ref", strict=True)
     for cm in references.get('commands', []):
@@ -99,7 +99,7 @@ def fix_single_reference(lines, cmd, aux, strict=True):
         rtxt = r['nicelabel']
         s = s[:i] + rtxt + s[j + 1:]
         i = i + len(rtxt)
-        print(cmd, rtxt)
+        # print(cmd, rtxt)
 
     lines = s.splitlines(keepends=False)
     return lines
\ No newline at end of file

src/snipper/fix_i.py

@@ -14,7 +14,7 @@ def rsession(analyzer, lines, extra):
     dbug = False
     # analyzer = we.spawn("python", encoding="utf-8", timeout=20)
     # analyzer.expect([">>>"])
-    if "BetterBasicDog" in "\n".join(lines) and False:
+    if "You can group" in "\n".join(lines): # in "\n".join(lines):
         print("\n".join(lines))
         print("-"*50)
         for k in extra['session_results']:
@@ -23,20 +23,27 @@ def rsession(analyzer, lines, extra):
 
         import time
         an = we.spawn("python", encoding="utf-8", timeout=20)
+        an.setwinsize(400, 400) # set window size to avoid truncated output or input.
         an.expect([">>>"])
         l3 = """
-import numpy as np 
-from scipy.linalg import norm
-x = np.asarray([3, 4])
-x  # What is x?
-norm(x)"""
+2 + 4 # Add two integers
+50 - 5 * 6
+(2 + 2) * (3 - 1) # You can group operations using parenthesis
+width = 20 # Assign the variable 'width' to a value of 20
+height = 5*9 # And also height is assigned the value of 5 * 9 = 45
+area = 2*3 # Compute the area of a rectangle and assign it to area now the text will be longer is that an issue
+area # This line shows us the value of 'area' #!i=b
+"""
         lines2 = l3.strip().splitlines()
+        from collections import defaultdict
+        dd = defaultdict(list)
 
         for l in lines2:
-            an.sendline(l)
+            dd['code'].append(l)
+            an.sendline(l.rstrip())
             an.expect_exact([">>>", "..."])
-            print("INPUT", l)
-            print(">>>", an.before.strip())
+            dd["output"].append(an.before.strip())
+            # print(">>>", an.before.strip())
             if len(an.after.strip()) > 4:
                 print(">>>>>>>>>>>>> That was a long after?")
             # analyzer.be
@@ -44,6 +51,9 @@ norm(x)"""
         print('*' * 50)
         # analyzer = an
         dbug = True
+        import tabulate
+        print(tabulate.tabulate(dd, headers='keys'))
+
     lines = "\n".join(lines).replace("\r", "").splitlines()
 
     for i, l in enumerate(lines):
@@ -68,16 +78,15 @@ norm(x)"""
         while True:
             time.sleep(0.05)
             analyzer.expect_exact([">>>", "..."])
-            if dbug and "total_cost" in word:
-                aaa = 23234
+            # if dbug and "total_cost" in word:
+            #     aaa = 23234
             before += analyzer.before
             # if dbug:
-            print(">  analyzer.before...", analyzer.before.strip(), "...AFTER...", analyzer.after.strip())
+            # print(">  analyzer.before...", analyzer.before.strip(), "...AFTER...", analyzer.after.strip())
             # AFTER =
             if analyzer.before.endswith("\n"):
-                print("> BREAKING LOOP")
+                # print("> BREAKING LOOP")
                 break
-                pass
             else:
                 before += analyzer.after
             break
@@ -108,6 +117,8 @@ norm(x)"""
 
 
 def run_i(lines, file, output):
+    if 'python0A' in str(file):
+        print(234)
     extra = dict(python=None, output=output, evaluated_lines=0, session_results=[])
     def block_fun(lines, start_extra, end_extra, art, head="", tail="", output=None, extra=None):
         outf = output + ("_" + art if art is not None and len(art) > 0 else "") + ".shell"
@@ -119,6 +130,7 @@ def run_i(lines, file, output):
 
         if extra['python'] is None:
             an = we.spawn("python", encoding="utf-8", timeout=20)
+            an.setwinsize(400, 400)  # set window size to avoid truncated output or input.
             an.expect([">>>"])
             extra['python'] = an
 
@@ -141,8 +153,8 @@ def run_i(lines, file, output):
             for outf in kvs:
                 out = "\n".join( ["\n".join(v[1]) for v in c if v[0] == outf] )
                 out = out.replace("\r", "")
-                if outf.endswith("python0B_e4.shell"):
-                    print(outf)
+                # if outf.endswith("python0B_e4.shell"):
+                #     print(outf)
 
                 with open(outf, 'w') as f:
                     f.write(out)

src/snipper/fix_s.py

@@ -17,8 +17,9 @@ def get_s(lines):
             pass
         if 'dse' in c['start_tag_args']:
             print("asdfasdfs")
-        print(c['start_tag_args'])
+        # print(c['start_tag_args'])
         if 'nodoc' in c['start_tag_args'] and c['start_tag_args']['nodoc']:
+            c['block'] = rm_docstring(c['block'])
             print("No documentation!")
         blocks[c['name']].append(c)
 
@@ -48,6 +49,8 @@ def get_s(lines):
     return output
 
 def rm_docstring(lines):
+    slines = lines
+    slines = dedent(lines)
     source = "\n".join(slines)
     import ast
     node = ast.parse(source)
@@ -58,7 +61,7 @@ def rm_docstring(lines):
     def rm_ds(f, ll2):
         if slines[f.lineno].strip().startswith('"' * 3):
             # print("got a docstrnig")
-            for k in range(f.lineno, f.end_lineno + 1):
+            for k in range(f.lineno-1, f.end_lineno + 1):
                 l = slines[k] if k != f.lineno else slines[k].strip()[3:]
                 if l.find('"' * 3) >= 0:
                     break
@@ -74,7 +77,18 @@ def rm_docstring(lines):
     for c in classes:
         for f in c.body:
             rm_ds(f, ll2)
-    return [l for l in ll2 if l is not None]
+        rm_ds(c, ll2)
+    nodoc = [l for l in ll2 if l is not None]
+    # print("\n".join(nodoc))
+    return nodoc
+
+
+def dedent(lines):
+    ll = lines
+    id = [indent(l) for l in ll if len(l.strip()) > 0]
+    id_len = [len(i) for i in id]
+    mindex = id_len[id_len.index(min(id_len))]
+    return [l[mindex:] for l in ll]
 
 
 # def _s_block_process():

src/snipper/load_citations.py

@@ -30,9 +30,9 @@ def get_aux(auxfile):
         if not os.path.dirname(auxdir):
             print("Directory for auxfile does not exist:", auxdir)
         else:
-            print("Auxfile did not exist, but directory did. Printing the contents:")
-            for f in glob.glob(os.path.dirname(auxfile) + "/*"):
-                print(f)
+            print("Auxfile did not exist, but directory did. Probably missing latex compile command.")
+            # for f in glob.glob(os.path.dirname(auxfile) + "/*"):
+            #     print(f)
 
         return {}
 

src/snipper/snip_dir.py

@@ -20,7 +20,7 @@ def snip_dir(source_dir,  # Sources
              verbose=True,
              ):
 
-    print("Snipper fixing", source_dir)
+    print(f"Snipper fixing {source_dir} {cut_files=}, {censor_files=}, {output_dir=}")
     if dest_dir == None:
         dest_dir = tempfile.mkdtemp()
         print("[snipper]", "no destination dir was specified so using nonsense destination:", dest_dir)
@@ -70,18 +70,7 @@ def snip_dir(source_dir,  # Sources
     for f, accept in acceptable:
         if os.path.isdir(f) or str(f).endswith("_grade.py"):
             continue
-        # if os.path.isdir(f) or not str(f).endswith(".py") or str(f).endswith("_grade.py"):
-        #     continue
-        # if accept and str(f).endswith(".rst"):
-        #     nrem, cut = censor_file(f, run_files=run_files, run_out_dirs=output_dir, cut_files=cut_files,
-        #                             # solution_list=solution_list,
-        #                             base_path=dest_dir,
-        #                             references=references,
-        #                             license_head=license_head,
-        #                             censor_files=censor_files,
-        #                             **kwargs)
 
-            # print("rst file")
         if accept and (str(f).endswith(".py") or str(f).endswith(".rst")):
             # if f.endswith("rst"):
             #     pass
@@ -109,5 +98,5 @@ def snip_dir(source_dir,  # Sources
             else:
                 if os.path.isdir(rm_file+"/"):
                     shutil.rmtree(rm_file)
-    print("[snipper]", "done")
+    # print("[snipper]", "done")
     return n, cutouts

src/snipper/snipper_main.py

@@ -48,8 +48,8 @@ def censor_file(file, run_files=True, run_out_dirs=None, cut_files=True,
         s = s.lstrip()
         lines = s.split("\n")
         for k, l in enumerate(lines):
-            if l.find(" # !") > 0:
-                print(f"{file}:{k}> bad snipper tag, fixing")
+            # if l.find(" # !") > 0:
+            #     print(f"{file}:{k}> bad snipper tag, fixing")
             lines[k] = l.replace("# !", "#!")
 
         try:

src/snipper/version.py

-__version__ = "0.1.15"
+__version__ = "0.1.17"

tests/demo1_correct/output/tree_a.shell

->>>
 >>> for _ in range(10):
 ...     print("hi")
+... 
+hi
+hi
+hi
+hi
+hi
+hi
+hi
+hi
+hi
+hi
\ No newline at end of file

tests/demo1_correct/output/tree_a_stripped.py

+# tree.py
+class TreeNode(object): 
+    def __init__(self, x): 
+
+def sorted_array_to_bst(nums):
+    if not nums: 
+        return None 
+    mid_val = len(nums) // 2 
\ No newline at end of file

tests/demo1_correct/output/tree_b.shell

->>>
->>>
 >>> print("hello")
 hello
 >>> def myfun(a):
 ...     return a*2
 ...
+>>>
+>>> print(myfun(4))
+8
\ No newline at end of file

tests/demo1_correct/output/tree_myfile_stripped.py

+        # tree.py
+        return None 
+    mid_val = len(nums) // 2 
+    node = TreeNode(nums[mid_val]) 
+    node.left = sorted_array_to_bst(nums[:mid_val])
+    node.right = sorted_array_to_bst(nums[mid_val + 1:]) 
+    return node 
+print("hello world asdfasd")
+
+#!o=a
+def preOrder(node):
+    if not node: 
+        return
+    print(node.val)
+    preOrder(node.left)
\ No newline at end of file

tests/demo1_tmp/output/tree_a.shell

->>>
 >>> for _ in range(10):
 ...     print("hi")
+... 
+hi
+hi
+hi
+hi
+hi
+hi
+hi
+hi
+hi
+hi
\ No newline at end of file

tests/demo1_tmp/output/tree_a_stripped.py

+# tree.py
+class TreeNode(object): 
+    def __init__(self, x): 
+
+def sorted_array_to_bst(nums):
+    if not nums: 
+        return None 
+    mid_val = len(nums) // 2 
\ No newline at end of file

tests/demo1_tmp/output/tree_b.shell

->>>
->>>
 >>> print("hello")
 hello
 >>> def myfun(a):
 ...     return a*2
 ...
+>>>
+>>> print(myfun(4))
+8
\ No newline at end of file

tests/demo1_tmp/output/tree_myfile_stripped.py

+        # tree.py
+        return None 
+    mid_val = len(nums) // 2 
+    node = TreeNode(nums[mid_val]) 
+    node.left = sorted_array_to_bst(nums[:mid_val])
+    node.right = sorted_array_to_bst(nums[mid_val + 1:]) 
+    return node 
+print("hello world asdfasd")
+
+#!o=a
+def preOrder(node):
+    if not node: 
+        return
+    print(node.val)
+    preOrder(node.left)
\ No newline at end of file