implement ch-image modify

bin/ch-completion.bash

@@ -259,14 +259,16 @@ _ch_convert_complete () {
 _image_build_opts="-b --bind --build-arg -f --file --force
                    --force-cmd -n --dry-run --parse-only -t --tag"
 
+_image_modify_opts="-o --out"
+
 _image_common_opts="-a --arch --always-download --auth --break
                     --cache --cache-large --dependencies -h --help
                     --no-cache --no-lock --no-xattrs --profile
                     --rebuild --password-many -q --quiet -s --storage
                     --tls-no-verify -v --verbose --version --xattrs"
 
-_image_subcommands="build build-cache delete gestalt
-                    import list pull push reset undelete"
+_image_subcommands="build build-cache delete gestalt import
+                    list modify pull push reset undelete"
 
 # archs taken from ARCH_MAP in charliecloud.py
 _archs="amd64 arm/v5 arm/v6 arm/v7 arm64/v8 386 mips64le ppc64le s390x"
@@ -374,20 +376,32 @@ _ch_image_complete () {
     build-cache)
         COMPREPLY=( $(compgen -W "--reset --gc --tree --dot" -- "$cur") )
         ;;
-    delete|list)
-        if [[ "$sub_cmd" == "list" ]]; then
+    delete|list|modify)
+        case "$sub_cmd" in
+        list)
             if [[ "$prev" == "--undeletable" || "$prev" == "--undeleteable" || "$prev" == "-u" ]]; then
                 COMPREPLY=( $(compgen -W "$(_ch_undelete_list "$strg_dir")" -- "$cur") )
                 return 0
             fi
-            extras+="$extras -l --long -u --undeletable"
+            extras="$extras -l --long -u --undeletable"
             # If “cur” starts with “--undelete,” add “--undeleteable” (the less
             # correct version of “--undeletable”) to the list of possible
             # completions.
             if [[ ${cur::10} == "--undelete" ]]; then
                 extras="$extras --undeleteable"
             fi
-        fi
+            ;;
+        modify)
+            case "$prev" in
+            -o|--out)
+                # Can’t complete for this option
+                COMPREPLY=()
+                return 0
+                ;;
+            esac
+            extras="$extras $_image_modify_opts"
+            ;;
+        esac
         COMPREPLY=( $(compgen -W "$(_ch_list_images "$strg_dir") $extras" -- "$cur") )
         __ltrim_colon_completions "$cur"
         ;;

bin/ch-image.py.in

@@ -13,6 +13,7 @@ import charliecloud as ch
 import build
 import build_cache as bu
 import filesystem as fs
+import image as im
 import misc
 import pull
 import push
@@ -278,6 +279,14 @@ def main():
    sp.add_argument("image_ref", metavar="IMAGE_REF", nargs="?",
                    help="print details of this image only")
 
+   # modify
+   sp = ap.add_parser("modify", "foo")
+   add_opts(sp, build.modify, deps_check=True, stog_init=True)
+   sp.add_argument("-c", metavar="CMD", action="append", default=[], nargs="*", help="foo")
+   sp.add_argument("-o", "--out", metavar="out_image", help="foo", required=True)
+   sp.add_argument("-S", "--shell", metavar="shell", help="foo")
+   sp.add_argument("image_ref", metavar="IMAGE_REF", help="image to modify")
+
    # pull
    sp = ap.add_parser("pull",
                       "copy image from remote repository to local filesystem")

doc/ch-image.rst

@@ -1895,6 +1895,80 @@ functionally identical when re-imported.
    will still hit, even if the new image is different.
 
 
+:code:`modify`
+==============
+
+Interactively edit the specified image.
+
+Synopsis
+--------
+
+::
+
+   $ ch-image [...] modify [...] TARGET
+
+Description
+-----------
+
+This subcommand starts a shell on the image named :code:`TARGET`, in order to
+edit the image interactively. It is similar to a :code:`RUN` instruction that
+starts an interactive shell. By default, ask the user whether to save changes
+when the shell exits.
+
+Options:
+
+  :code:`-m MSG`
+    Use :code:`MSG` to identify the edits to the build cache. That is, if you
+    run this command twice with the same :code:`TARGET`, the same :code:`-o
+    DEST`, and the same :code:`MSG`, the second session will overwrite the
+    first. (Without :code:`-o`, the second session will build atop the first.)
+    By default, every interactive session is considered different from every
+    other, as if a random :code:`MSG` were entered.
+
+  :code:`-o`, :code:`--out DEST`
+    Save the results in image named :code:`DEST`, leaving :code:`TARGET`
+    unchanged.
+
+  :code:`-s`, :code:`--shell SHELL`
+    Start :code:`SHELL` instead of :code:`/bin/sh`.
+
+  :code:`-y`, :code:`--yes`
+    Do not prompt the user to save. Instead, save if the shell exits
+    successfully, and roll back if it exits unsuccessfully, e.g. by executing
+    :code:`exit 1`.
+
+.. warning::
+
+   This subcommand is rarely needed. Non-interactive build using a Dockerfile
+   is almost always better, because it preserves the sequence of operations
+   that created an image. Only use this subcommand if you really know what you
+   are doing.
+
+Examples
+--------
+
+To edit the image :code:`foo`, adding :code:`/opt/lib` to the default shared
+library search path, producing image :code:`bar` as the result::
+
+   $ ch-image modify -o bar foo
+   [...]
+   > emacs /etc/ld.so.conf
+   [... append line “/opt/lib” to the file ...]
+   > ldconfig
+   > exit
+   Save changes ([y]/n)? y
+   committing ...
+   [...]
+
+Equivalently, and almost certainly preferred::
+
+   $ cat Dockerfile
+   FROM foo
+   RUN echo /opt/lib >> /etc/ld.so.conf
+   RUN ldconfig
+   $ ch-image build -t bar -f Dockerfile .
+
+
 :code:`pull`
 ============
 

lib/build.py

@@ -9,6 +9,7 @@
 import os.path
 import re
 import shutil
+import subprocess
 import sys
 
 import charliecloud as ch
@@ -57,6 +58,10 @@ class Environment:
       of this is just passed through from the image metadata."""
 
 
+# Class responsible for traversing the parse tree generated by lark. “Main_Loop”
+# visits each node in the parse tree and calls its “__default__” method to
+# figure out what to do with the node. This behavior is defined by the parent
+# class, “lark.Visitor”, documented here: https://lark-parser.readthedocs.io/en/latest/visitors.html
 class Main_Loop(lark.Visitor):
 
    __slots__ = ("instruction_total_ct",
@@ -69,15 +74,28 @@ def __init__(self, *args, **kwargs):
       self.instruction_total_ct = 0
       super().__init__(*args, **kwargs)
 
+   # The main argument of the “__default__” method is “tree”, which is really
+   # just the current node being visited (the node is called “tree” because it
+   # represents the root node of the current subtree). The “tree.data” attribute
+   # gives the “name of the rule or alias” represented by the node. When a
+   # “lark.Visitor” instance visits a parse tree node, it checks the node’s
+   # “data” attribute and tries to call its own attribute (e.g. method) of the
+   # same name. If no such attribute exists, it calls “__default__”. Note that
+   # in this class, we don’t define any attributes corresponding to the
+   # Dockerfile instructions that we want to execute, so “__default__” is always
+   # called when visiting a node. We rely on instruction classes to execute the
+   # instructions, rather than attributes of this class.
    def __default__(self, tree):
       class_ = tree.data.title() + "_G"
+      ch.ILLERI(tree.data.title())
       if (class_ in globals()):
          inst = globals()[class_](tree)
          if (self.instruction_total_ct == 0):
             if (not (isinstance(inst, Directive_G)
                   or isinstance(inst, From__G)
                   or isinstance(inst, Instruction_No_Image))):
                ch.FATAL("first instruction must be ARG or FROM")
+         ch.ILLERI("PREV: %s" % self.inst_prev)
          inst.init(self.inst_prev)
          # The three announce_maybe() calls are clunky but I couldn’t figure
          # out how to avoid the repeats.
@@ -287,6 +305,129 @@ def unescape(sl):
    assert (len(sl) >= 2 and sl[0] == '"' and sl[-1] == '"' and sl[-2:] != '\\"')
    return ast.literal_eval(sl)
 
+def modify(cli_):
+   global cli
+   cli = cli_
+
+   cli.parse_only = False
+   cli.force = ch.Force_Mode.SECCOMP
+   cli.force_cmd = force.FORCE_CMD_DEFAULT
+   cli.bind = []
+   # FIXME: This is super kludgey
+   cli.tag = cli.out
+
+   print(cli.image_ref)
+   ch.ILLERI(cli.c)
+   ch.ILLERI(type(cli.c))
+   commands = []
+   # “Flatten” commands array
+   for c in cli.c:
+      commands += c
+   src_image = im.Image(im.Reference(cli.image_ref))
+   out_image = cli.out
+   if (not src_image.unpack_exist_p):
+      ch.FATAL("not in storage: %s" % src_image.ref)
+   if (out_image == str(src_image.ref)):
+      ch.FATAL("output image must have different name from source (%s)" % src_image.ref)
+   if (cli.shell is not None):
+      shell = cli.shell
+   else:
+      shell = "/bin/sh"
+   if not sys.stdin.isatty():
+      # https://stackoverflow.com/a/17735803
+      # commands from stdin
+      for line in sys.stdin:
+         # execute each line (analogous to RUN)
+         commands.append(line)
+   if (commands != []):
+      tree = modify_tree_make(src_image.ref, commands)
+
+      # FIXME: Be more DRY in this section
+
+      # Count the number of stages (i.e., FROM instructions)
+      global image_ct
+      image_ct = sum(1 for i in tree.children_("from_"))
+
+      ml = Main_Loop()
+      if (hasattr(ml, 'visit_topdown')):
+         ml.visit_topdown(tree)
+      else:
+         ml.visit(tree)
+      if (ml.instruction_total_ct > 0):
+         if (ml.miss_ct == 0):
+            ml.inst_prev.checkout()
+         ml.inst_prev.ready()
+
+   else:
+      # Make sure that shell exists.
+      #try:
+      #   subprocess.run([ch.CH_BIN + "/ch-run", str(src_image.ref), "--", shell], capture_output=True).check_returncode()
+      #except subprocess.CalledProcessError as x:
+      #   #print(x.__dict__)
+      #   if ("%s: No such file or directory" % shell in str(x.stderr)):
+      #      ch.FATAL("invalid shell: %s" % shell)
+
+      out_image = im.Image(im.Reference(out_image))
+      # Do something similar to “ch-image import”
+      out_image.unpack_clear()
+      out_image.copy_unpacked(src_image)
+      subprocess.run([ch.CH_BIN + "/ch-run", "--unsafe", "-w",
+                      str(out_image.ref), "--", shell])
+      # FIXME: metadata history stuff? See misc.import_.
+      #bu.cache.rollback(src_image.unpack_path)
+
+def modify_tree_make(src_img, cmds):
+   """Function that manually constructs a parse tree corresponding to a set of
+      “ch-image modify” commands, as though the commands had been specified in a
+      Dockerfile. Note that because “ch-image modify” simply executes one or
+      more commands inside a container, the only Dockerfile instructions we need
+      to consider are “FROM” and “RUN”. This results in conveniently simple
+      parse trees of the form:
+
+         start
+            dockerfile
+               from_
+                  image_ref
+                     IMAGE_REF <image_name>
+               run
+                  run_shell
+                     LINE_CHUNK <command_1>
+               [...]
+               run
+                  run_shell
+                     LINE_CHUNK <command_N>
+
+      e.g. for the command line
+
+         $ ch-image modify -o foo2 -c 'echo foo' -c 'echo bar' -- foo
+
+      this function produces the following parse tree
+
+         start
+            dockerfile
+               from_
+                  image_ref
+                     IMAGE_REF foo
+               run
+                  run_shell
+                     LINE_CHUNK echo foo
+               run
+                  run_shell
+                     LINE_CHUNK echo bar
+      """
+   # Children of dockerfile tree
+   df_children = []
+   # Metadata attribute. We use this attribute in the “_pretty” method for our
+   # “Tree” class. Constructing a tree without specifying a “Meta” instance that
+   # has been given a “line” value will result in the attribute not being present,
+   # which causes an error when we try to access that attribute. Here we give the
+   # attribute a debug value of -1 to avoid said errors.
+   meta = lark.tree.Meta()
+   meta.line = -1
+   df_children.append(im.Tree(lark.Token('RULE', 'from_'), [im.Tree(lark.Token('RULE', 'image_ref'),[lark.Token('IMAGE_REF', src_img.name)], meta)], meta))
+   for cmd in cmds:
+      df_children.append(im.Tree(lark.Token('RULE', 'run'), [im.Tree(lark.Token('RULE', 'run_shell'),[lark.Token('LINE_CHUNK', cmd)], meta)],meta))
+   return im.Tree(lark.Token('RULE', 'start'), [im.Tree(lark.Token('RULE','dockerfile'), df_children)], meta)
 
 ## Supporting classes ##
 

lib/image.py

@@ -4,9 +4,11 @@
 import json
 import os
 import re
+import subprocess
 import sys
 import tarfile
 
+import build_cache as bu
 import charliecloud as ch
 import filesystem as fs
 
@@ -947,3 +949,4 @@ def terminals_cat(self, tname):
       """Return the concatenated values of all child terminals named tname as
          a string, with no delimiters. If none, return the empty string."""
       return "".join(self.terminals(tname))
+