From e8b04c9739230ed5ba6ad684496c38c3b3c80f5b Mon Sep 17 00:00:00 2001 From: Reid Priedhorsky Date: Fri, 8 Jul 2022 21:46:43 -0600 Subject: [PATCH 01/65] draft docs [skip ci] --- doc/ch-image_desc.rst | 74 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/doc/ch-image_desc.rst b/doc/ch-image_desc.rst index 6bc0e7470..22ef0c51e 100644 --- a/doc/ch-image_desc.rst +++ b/doc/ch-image_desc.rst @@ -856,6 +856,80 @@ functionally identical when re-imported. already been imported. +:code:`modify` +============== + +Interactively edit the specified image. + +Synopsis +-------- + +:: + + $ ch-image [...] modify [...] TARGET + +Description +----------- + +This subcommand starts a shell on the image named :code:`TARGET`, in order to +edit the image interactively. It is similar to a :code:`RUN` instruction that +starts an interactive shell. By default, ask the user whether to save changes +when the shell exits. + +Options: + + :code:`-m MSG` + Use :code:`MSG` to identify the edits to the build cache. That is, if you + run this command twice with the same :code:`TARGET`, the same :code:`-o + DEST`, and the same :code:`MSG`, the second session will overwrite the + first. (Without :code:`-o`, the second session will build atop the first.) + By default, every interactive session is considered different from every + other, as if a random :code:`MSG` were entered. + + :code:`-o`, :code:`--out DEST` + Save the results in image named :code:`DEST`, leaving :code:`TARGET` + unchanged. + + :code:`-s`, :code:`--shell SHELL` + Start :code:`SHELL` instead of :code:`/bin/sh`. + + :code:`-y`, :code:`--yes` + Do not prompt the user to save. Instead, save if the shell exits + successfully, and roll back if it exits unsuccessfully, e.g. by executing + :code:`exit 1`. + +.. warning:: + + This subcommand is rarely needed. Non-interactive build using a Dockerfile + is almost always better, because it preserves the sequence of operations + that created an image. Only use this subcommand if you really know what you + are doing. + +Examples +-------- + +To edit the image :code:`foo`, adding :code:`/opt/lib` to the default shared +library search path, producing image :code:`bar` as the result:: + + $ ch-image modify -o bar foo + [...] + > emacs /etc/ld.so.conf + [... append line “/opt/lib” to the file ...] + > ldconfig + > exit + Save changes ([y]/n)? y + committing ... + [...] + +Equivalently, and almost certainly preferred:: + + $ cat Dockerfile + FROM foo + RUN echo /opt/lib >> /etc/ld.so.conf + RUN ldconfig + $ ch-image build -t bar -f Dockerfile . + + :code:`pull` ============ From 83beda8daecb0e55ace60d10024b7514d26aa457 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Wed, 14 Feb 2024 18:49:59 +0000 Subject: [PATCH 02/65] add proper formatting [skip ci] --- doc/ch-image.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/doc/ch-image.rst b/doc/ch-image.rst index 9066cf70a..90da5dec3 100644 --- a/doc/ch-image.rst +++ b/doc/ch-image.rst @@ -1880,18 +1880,26 @@ functionally identical when re-imported. :code:`modify` ============== + Interactively edit the specified image. + Synopsis -------- + :: + $ ch-image [...] modify [...] TARGET + Description ----------- + This subcommand starts a shell on the image named :code:`TARGET`, in order to edit the image interactively. It is similar to a :code:`RUN` instruction that starts an interactive shell. By default, ask the user whether to save changes when the shell exits. + Options: + :code:`-m MSG` Use :code:`MSG` to identify the edits to the build cache. That is, if you run this command twice with the same :code:`TARGET`, the same :code:`-o @@ -1899,24 +1907,32 @@ Options: first. (Without :code:`-o`, the second session will build atop the first.) By default, every interactive session is considered different from every other, as if a random :code:`MSG` were entered. + :code:`-o`, :code:`--out DEST` Save the results in image named :code:`DEST`, leaving :code:`TARGET` unchanged. + :code:`-s`, :code:`--shell SHELL` Start :code:`SHELL` instead of :code:`/bin/sh`. + :code:`-y`, :code:`--yes` Do not prompt the user to save. Instead, save if the shell exits successfully, and roll back if it exits unsuccessfully, e.g. by executing :code:`exit 1`. + .. warning:: + This subcommand is rarely needed. Non-interactive build using a Dockerfile is almost always better, because it preserves the sequence of operations that created an image. Only use this subcommand if you really know what you are doing. + Examples -------- + To edit the image :code:`foo`, adding :code:`/opt/lib` to the default shared library search path, producing image :code:`bar` as the result:: + $ ch-image modify -o bar foo [...] > emacs /etc/ld.so.conf @@ -1926,7 +1942,9 @@ library search path, producing image :code:`bar` as the result:: Save changes ([y]/n)? y committing ... [...] + Equivalently, and almost certainly preferred:: + $ cat Dockerfile FROM foo RUN echo /opt/lib >> /etc/ld.so.conf From 244d17423b66f019a77b3e0ac44058412cb0fc7e Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Mon, 26 Feb 2024 16:51:16 +0000 Subject: [PATCH 03/65] interactive modify [skip ci] --- bin/ch-image.py.in | 7 +++++++ lib/image.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/bin/ch-image.py.in b/bin/ch-image.py.in index 0de6ca8f5..37353a139 100644 --- a/bin/ch-image.py.in +++ b/bin/ch-image.py.in @@ -12,6 +12,7 @@ import charliecloud as ch import build import misc import filesystem as fs +import image as im import pull import push @@ -273,6 +274,12 @@ def main(): sp.add_argument("image_ref", metavar="IMAGE_REF", nargs="?", help="print details of this image only") + # modify + sp = ap.add_parser("modify", "foo") + add_opts(sp, im.modify, deps_check=True, stog_init=True) + sp.add_argument("-o", "--out", metavar="out_image", help="foo") + sp.add_argument("image_ref", metavar="IMAGE_REF", help="image to modify") + # pull sp = ap.add_parser("pull", "copy image from remote repository to local filesystem") diff --git a/lib/image.py b/lib/image.py index 51d563a1a..0819f6774 100644 --- a/lib/image.py +++ b/lib/image.py @@ -4,9 +4,11 @@ import json import os import re +import subprocess import sys import tarfile +import build_cache as bu import charliecloud as ch import filesystem as fs @@ -920,3 +922,43 @@ def terminals_cat(self, tname): """Return the concatenated values of all child terminals named tname as a string, with no delimiters. If none, return the empty string.""" return "".join(self.terminals(tname)) + +## Functions ## + +def modify(cli): + print(cli.image_ref) + src_image = Image(Reference(cli.image_ref)) + print("OUT: %s" % cli.out) + out_image = cli.out + print("unpack %s" % src_image.unpack_path) + if (not src_image.unpack_exist_p): + ch.FATAL("not in storage: %s" % src_image.ref.name) + if (out_image == src_image.ref.name): + ch.FATAL("choose a different name brah") + # Make a temporary copy image to run our changes in. (I'm doing this to + # support rolling back changes in the case of a disabled build cache). + if not sys.stdin.isatty(): + # https://stackoverflow.com/a/17735803 + # commands from stdin + print("foo") + else: + print(src_image.ref.name) + subprocess.run([ch.CH_BIN + "/ch-run", "--unsafe", "-w", + str(src_image.ref), "--", "/bin/sh"]) + save = input("Save changes ([y]/n)? ") + if (save): + if (not out_image): + while True: + out_image = input("specify destination image: ") + if (out_image != src_image.ref.name): + break + else: + ch.ERROR("choose a different name brah") + out_image = Image(Reference(out_image)) + # Do something similar to “ch-image import” + out_image.unpack_clear() + out_image.copy_unpacked(src_image) + # FIXME: metadata history stuff? See misc.import_. + else: + pass + bu.cache.rollback(src_image.unpack_path) From 580b6ec0112aa370d2e4bb663a1e5cd9060326fe Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Thu, 14 Mar 2024 18:50:17 +0000 Subject: [PATCH 04/65] move modify function, other additions (buggy) [skip ci] --- bin/ch-image.py.in | 4 +- lib/build.py | 179 ++++++++++++++++++++++++++++++++++++--------- lib/image.py | 39 ---------- 3 files changed, 149 insertions(+), 73 deletions(-) diff --git a/bin/ch-image.py.in b/bin/ch-image.py.in index 25ed3b162..d468f0712 100644 --- a/bin/ch-image.py.in +++ b/bin/ch-image.py.in @@ -281,8 +281,10 @@ def main(): # modify sp = ap.add_parser("modify", "foo") - add_opts(sp, im.modify, deps_check=True, stog_init=True) + add_opts(sp, build.modify, deps_check=True, stog_init=True) + sp.add_argument("-c", metavar="CMD", action="append", default=[], nargs="*", help="foo") sp.add_argument("-o", "--out", metavar="out_image", help="foo") + sp.add_argument("-S", "--shell", metavar="shell", help="foo") sp.add_argument("image_ref", metavar="IMAGE_REF", help="image to modify") # pull diff --git a/lib/build.py b/lib/build.py index 49fdccc57..55eae3fd1 100644 --- a/lib/build.py +++ b/lib/build.py @@ -9,6 +9,7 @@ import os.path import re import shutil +import subprocess import sys import charliecloud as ch @@ -57,6 +58,10 @@ class Environment: of this is just passed through from the image metadata.""" +# Class responsible for traversing the parse tree generated by lark. “Main_Loop” +# visits each node in the parse tree and calls its “__default__” method to +# figure out what to do with the node. This behavior is defined by the parent +# class, “lark.Visitor”, documented here: https://lark-parser.readthedocs.io/en/latest/visitors.html class Main_Loop(lark.Visitor): __slots__ = ("instruction_total_ct", @@ -69,8 +74,20 @@ def __init__(self, *args, **kwargs): self.instruction_total_ct = 0 super().__init__(*args, **kwargs) + # The main argument of the “__default__” method is “tree”, which is really + # just the current node being visited (the node is called “tree” because it + # represents the root node of the current subtree). The “tree.data” attribute + # gives the “name of the rule or alias” represented by the node. When a + # “lark.Visitor” instance visits a parse tree node, it checks the node’s + # “data” attribute and tries to call its own attribute (e.g. method) of the + # same name. If no such attribute exists, it calls “__default__”. Note that + # in this class, we don’t define any attributes corresponding to the + # Dockerfile instructions that we want to execute, so “__default__” is always + # called when visiting a node. We rely on instruction classes to execute the + # instructions, rather than attributes of this class. def __default__(self, tree): class_ = tree.data.title() + "_G" + ch.ILLERI(tree.data.title()) if (class_ in globals()): inst = globals()[class_](tree) if (self.instruction_total_ct == 0): @@ -78,6 +95,7 @@ def __default__(self, tree): or isinstance(inst, From__G) or isinstance(inst, Instruction_No_Image))): ch.FATAL("first instruction must be ARG or FROM") + ch.ILLERI("PREV: %s" % self.inst_prev) inst.init(self.inst_prev) # The three announce_maybe() calls are clunky but I couldn’t figure # out how to avoid the repeats. @@ -198,6 +216,126 @@ def build_arg_get(arg): text = ch.ossafe("can’t read: %s" % cli.file, fp.read) ch.close_(fp) + ml = parse_n_traverse(text) + + # Check that all build arguments were consumed. + if (len(cli.build_arg) != 0): + ch.FATAL("--build-arg: not consumed: " + " ".join(cli.build_arg.keys())) + + # Print summary & we’re done. + if (ml.instruction_total_ct == 0): + ch.FATAL("no instructions found: %s" % cli.file) + assert (ml.inst_prev.image_i + 1 == image_ct) # should’ve errored already + if ((cli.force != ch.Force_Mode.NONE) and ml.miss_ct != 0): + ch.INFO("--force=%s: modified %d RUN instructions" + % (cli.force.value, forcer.run_modified_ct)) + ch.INFO("grown in %d instructions: %s" + % (ml.instruction_total_ct, ml.inst_prev.image)) + # FIXME: remove when we’re done encouraging people to use the build cache. + if (isinstance(bu.cache, bu.Disabled_Cache)): + ch.INFO("build slow? consider enabling the build cache", + "https://hpc.github.io/charliecloud/command-usage.html#build-cache") + + +## Functions ## + +def unescape(sl): + # FIXME: This is also ugly and should go in the grammar. + # + # The Dockerfile spec does not precisely define string escaping, but I’m + # guessing it’s the Go rules. You will note that we are using Python rules. + # This is wrong but close enough for now (see also gripe in previous + # paragraph). + if ( not sl.startswith('"') # no start quote + and (not sl.endswith('"') or sl.endswith('\\"'))): # no end quote + sl = '"%s"' % sl + assert (len(sl) >= 2 and sl[0] == '"' and sl[-1] == '"' and sl[-2:] != '\\"') + return ast.literal_eval(sl) + +def modify(cli_): + global cli + cli = cli_ + + cli.parse_only = False + cli.force = ch.Force_Mode.SECCOMP + cli.force_cmd = force.FORCE_CMD_DEFAULT + cli.bind = [] + + print(cli.image_ref) + ch.ILLERI(cli.c) + ch.ILLERI(type(cli.c)) + commands = [] + # “Flatten” commands array + for c in cli.c: + ch.ILLERI(c) + commands += c + src_image = im.Image(im.Reference(cli.image_ref)) + print("OUT: %s" % cli.out) + out_image = cli.out + cli.tag = cli.out + print("unpack %s" % src_image.unpack_path) + if (not src_image.unpack_exist_p): + ch.FATAL("not in storage: %s" % src_image.ref.name) + if (out_image == src_image.ref.name): + ch.FATAL("choose a different name brah") + if (cli.shell is not None): + shell = cli.shell + else: + shell = "/bin/sh" + # Make a temporary copy image to run our changes in. (I'm doing this to + # support rolling back changes in the case of a disabled build cache). + if not sys.stdin.isatty(): + # https://stackoverflow.com/a/17735803 + # commands from stdin + print("foo") + ch.ILLERI("INPUT DETECTED") + for line in sys.stdin: + # execute each line (analogous to RUN) + ch.ILLERI(line) + elif (cli.c != []): + print("bar") + df = "FROM %s\n" % src_image.ref.name + for cmd in commands: + df += "RUN %s\n" % cmd + df += "\n" + ml = parse_n_traverse(df) + + else: + #print(src_image.ref.name) + ch.ILLERI(src_image.ref) + # Make sure that shell exists. + #try: + # subprocess.run([ch.CH_BIN + "/ch-run", str(src_image.ref), "--", shell], capture_output=True).check_returncode() + #except subprocess.CalledProcessError as x: + # #print(x.__dict__) + # if ("%s: No such file or directory" % shell in str(x.stderr)): + # ch.FATAL("invalid shell: %s" % shell) + + subprocess.run([ch.CH_BIN + "/ch-run", "--unsafe", "-w", + str(src_image.ref), "--", shell]) + save = input("Save changes ([y]/n)? ") + if (save.lower() in ["y", "yes"]): + if (not out_image): + while True: + out_image = input("specify destination image: ") + if (out_image != src_image.ref.name): + break + else: + ch.ERROR("choose a different name brah") + out_image = im.Image(im.Reference(out_image)) + # Do something similar to “ch-image import” + out_image.unpack_clear() + out_image.copy_unpacked(src_image) + # FIXME: metadata history stuff? See misc.import_. + else: + pass + bu.cache.rollback(src_image.unpack_path) + + +# FIXME: For some reason, on this branch the parsed tree node coresponding to +# “alpine:latest” is just called “alpine”. This does not happen on the +# master branch, so I need to get to the bottom of this. +def parse_n_traverse(text): # Parse it. parser = lark.Lark(im.GRAMMAR_DOCKERFILE, parser="earley", propagate_positions=True, tree_class=im.Tree) @@ -253,40 +391,15 @@ def build_arg_get(arg): ml.inst_prev.checkout() ml.inst_prev.ready() - # Check that all build arguments were consumed. - if (len(cli.build_arg) != 0): - ch.FATAL("--build-arg: not consumed: " + " ".join(cli.build_arg.keys())) - - # Print summary & we’re done. - if (ml.instruction_total_ct == 0): - ch.FATAL("no instructions found: %s" % cli.file) - assert (ml.inst_prev.image_i + 1 == image_ct) # should’ve errored already - if ((cli.force != ch.Force_Mode.NONE) and ml.miss_ct != 0): - ch.INFO("--force=%s: modified %d RUN instructions" - % (cli.force.value, forcer.run_modified_ct)) - ch.INFO("grown in %d instructions: %s" - % (ml.instruction_total_ct, ml.inst_prev.image)) - # FIXME: remove when we’re done encouraging people to use the build cache. - if (isinstance(bu.cache, bu.Disabled_Cache)): - ch.INFO("build slow? consider enabling the build cache", - "https://hpc.github.io/charliecloud/command-usage.html#build-cache") - - -## Functions ## - -def unescape(sl): - # FIXME: This is also ugly and should go in the grammar. - # - # The Dockerfile spec does not precisely define string escaping, but I’m - # guessing it’s the Go rules. You will note that we are using Python rules. - # This is wrong but close enough for now (see also gripe in previous - # paragraph). - if ( not sl.startswith('"') # no start quote - and (not sl.endswith('"') or sl.endswith('\\"'))): # no end quote - sl = '"%s"' % sl - assert (len(sl) >= 2 and sl[0] == '"' and sl[-1] == '"' and sl[-2:] != '\\"') - return ast.literal_eval(sl) + return ml +def modify_tree_make(src_img, cmds): + # Children of dockerfile tree + #df_children = [] + src_img_ref = im.Reference(src_img) + ch.ILLERI(src_img) + #lark.Tree(lark.Token('RULE', 'image_ref')[lark.Token('IMAGE_REF', src_img_ref)]) + #df_children.append() ## Supporting classes ## diff --git a/lib/image.py b/lib/image.py index cb80a1f58..d1038fe5a 100644 --- a/lib/image.py +++ b/lib/image.py @@ -950,42 +950,3 @@ def terminals_cat(self, tname): a string, with no delimiters. If none, return the empty string.""" return "".join(self.terminals(tname)) -## Functions ## - -def modify(cli): - print(cli.image_ref) - src_image = Image(Reference(cli.image_ref)) - print("OUT: %s" % cli.out) - out_image = cli.out - print("unpack %s" % src_image.unpack_path) - if (not src_image.unpack_exist_p): - ch.FATAL("not in storage: %s" % src_image.ref.name) - if (out_image == src_image.ref.name): - ch.FATAL("choose a different name brah") - # Make a temporary copy image to run our changes in. (I'm doing this to - # support rolling back changes in the case of a disabled build cache). - if not sys.stdin.isatty(): - # https://stackoverflow.com/a/17735803 - # commands from stdin - print("foo") - else: - print(src_image.ref.name) - subprocess.run([ch.CH_BIN + "/ch-run", "--unsafe", "-w", - str(src_image.ref), "--", "/bin/sh"]) - save = input("Save changes ([y]/n)? ") - if (save): - if (not out_image): - while True: - out_image = input("specify destination image: ") - if (out_image != src_image.ref.name): - break - else: - ch.ERROR("choose a different name brah") - out_image = Image(Reference(out_image)) - # Do something similar to “ch-image import” - out_image.unpack_clear() - out_image.copy_unpacked(src_image) - # FIXME: metadata history stuff? See misc.import_. - else: - pass - bu.cache.rollback(src_image.unpack_path) From e8435764e15617600211af832bfa64a519d9cdb1 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Tue, 19 Mar 2024 17:04:30 +0000 Subject: [PATCH 05/65] clean stuff up a bit? --- bin/ch-image.py.in | 2 +- lib/build.py | 242 +++++++++++++++++++++++++-------------------- 2 files changed, 136 insertions(+), 108 deletions(-) diff --git a/bin/ch-image.py.in b/bin/ch-image.py.in index d468f0712..dd64a8a7c 100644 --- a/bin/ch-image.py.in +++ b/bin/ch-image.py.in @@ -283,7 +283,7 @@ def main(): sp = ap.add_parser("modify", "foo") add_opts(sp, build.modify, deps_check=True, stog_init=True) sp.add_argument("-c", metavar="CMD", action="append", default=[], nargs="*", help="foo") - sp.add_argument("-o", "--out", metavar="out_image", help="foo") + sp.add_argument("-o", "--out", metavar="out_image", help="foo", required=True) sp.add_argument("-S", "--shell", metavar="shell", help="foo") sp.add_argument("image_ref", metavar="IMAGE_REF", help="image to modify") diff --git a/lib/build.py b/lib/build.py index 55eae3fd1..f7ff53fc9 100644 --- a/lib/build.py +++ b/lib/build.py @@ -216,7 +216,60 @@ def build_arg_get(arg): text = ch.ossafe("can’t read: %s" % cli.file, fp.read) ch.close_(fp) - ml = parse_n_traverse(text) + # Parse it. + parser = lark.Lark(im.GRAMMAR_DOCKERFILE, parser="earley", + propagate_positions=True, tree_class=im.Tree) + # Avoid Lark issue #237: lark.exceptions.UnexpectedEOF if the file does not + # end in newline. + text += "\n" + try: + tree = parser.parse(text) + except lark.exceptions.UnexpectedInput as x: + ch.VERBOSE(x) # noise about what was expected in the grammar + ch.FATAL("can’t parse: %s:%d,%d\n\n%s" + % (cli.file, x.line, x.column, x.get_context(text, 39))) + ch.VERBOSE(tree.pretty()[:-1]) # rm trailing newline + + # Sometimes we exit after parsing. + if (cli.parse_only): + ch.exit(0) + + # Count the number of stages (i.e., FROM instructions) + global image_ct + image_ct = sum(1 for i in tree.children_("from_")) + + # If we use RSYNC, error out quickly if appropriate rsync(1) not present. + if (tree.child("rsync") is not None): + try: + ch.version_check(["rsync", "--version"], ch.RSYNC_MIN) + except ch.Fatal_Error: + ch.ERROR("Dockerfile uses RSYNC, so rsync(1) is required") + raise + + # Traverse the tree and do what it says. + # + # We don’t actually care whether the tree is traversed breadth-first or + # depth-first, but we *do* care that instruction nodes are visited in + # order. Neither visit() nor visit_topdown() are documented as of + # 2020-06-11 [1], but examining source code [2] shows that visit_topdown() + # uses Tree.iter_trees_topdown(), which *is* documented to be in-order [3]. + # + # This change seems to have been made in 0.8.6 (see PR #761); before then, + # visit() was in order. Therefore, we call that instead, if visit_topdown() + # is not present, to improve compatibility (see issue #792). + # + # [1]: https://lark-parser.readthedocs.io/en/latest/visitors/#visitors + # [2]: https://github.com/lark-parser/lark/blob/445c8d4/lark/visitors.py#L211 + # [3]: https://lark-parser.readthedocs.io/en/latest/classes/#tree + ml = Main_Loop() + if (hasattr(ml, 'visit_topdown')): + ml.visit_topdown(tree) + else: + ml.visit(tree) + if (ml.instruction_total_ct > 0): + if (ml.miss_ct == 0): + ml.inst_prev.checkout() + ml.inst_prev.ready() # Check that all build arguments were consumed. if (len(cli.build_arg) != 0): @@ -260,6 +313,8 @@ def modify(cli_): cli.force = ch.Force_Mode.SECCOMP cli.force_cmd = force.FORCE_CMD_DEFAULT cli.bind = [] + # FIXME: This is super kludgey + cli.tag = cli.out print(cli.image_ref) ch.ILLERI(cli.c) @@ -267,42 +322,43 @@ def modify(cli_): commands = [] # “Flatten” commands array for c in cli.c: - ch.ILLERI(c) commands += c src_image = im.Image(im.Reference(cli.image_ref)) - print("OUT: %s" % cli.out) out_image = cli.out - cli.tag = cli.out - print("unpack %s" % src_image.unpack_path) if (not src_image.unpack_exist_p): - ch.FATAL("not in storage: %s" % src_image.ref.name) - if (out_image == src_image.ref.name): - ch.FATAL("choose a different name brah") + ch.FATAL("not in storage: %s" % src_image.ref) + if (out_image == str(src_image.ref)): + ch.FATAL("output image must have different name from source (%s)" % src_image.ref) if (cli.shell is not None): shell = cli.shell else: shell = "/bin/sh" - # Make a temporary copy image to run our changes in. (I'm doing this to - # support rolling back changes in the case of a disabled build cache). if not sys.stdin.isatty(): # https://stackoverflow.com/a/17735803 # commands from stdin - print("foo") - ch.ILLERI("INPUT DETECTED") for line in sys.stdin: # execute each line (analogous to RUN) - ch.ILLERI(line) - elif (cli.c != []): - print("bar") - df = "FROM %s\n" % src_image.ref.name - for cmd in commands: - df += "RUN %s\n" % cmd - df += "\n" - ml = parse_n_traverse(df) + commands.append(line) + if (commands != []): + tree = modify_tree_make(src_image.ref, commands) + + # FIXME: Be more DRY in this section + + # Count the number of stages (i.e., FROM instructions) + global image_ct + image_ct = sum(1 for i in tree.children_("from_")) + + ml = Main_Loop() + if (hasattr(ml, 'visit_topdown')): + ml.visit_topdown(tree) + else: + ml.visit(tree) + if (ml.instruction_total_ct > 0): + if (ml.miss_ct == 0): + ml.inst_prev.checkout() + ml.inst_prev.ready() else: - #print(src_image.ref.name) - ch.ILLERI(src_image.ref) # Make sure that shell exists. #try: # subprocess.run([ch.CH_BIN + "/ch-run", str(src_image.ref), "--", shell], capture_output=True).check_returncode() @@ -311,95 +367,67 @@ def modify(cli_): # if ("%s: No such file or directory" % shell in str(x.stderr)): # ch.FATAL("invalid shell: %s" % shell) + out_image = im.Image(im.Reference(out_image)) + # Do something similar to “ch-image import” + out_image.unpack_clear() + out_image.copy_unpacked(src_image) subprocess.run([ch.CH_BIN + "/ch-run", "--unsafe", "-w", - str(src_image.ref), "--", shell]) - save = input("Save changes ([y]/n)? ") - if (save.lower() in ["y", "yes"]): - if (not out_image): - while True: - out_image = input("specify destination image: ") - if (out_image != src_image.ref.name): - break - else: - ch.ERROR("choose a different name brah") - out_image = im.Image(im.Reference(out_image)) - # Do something similar to “ch-image import” - out_image.unpack_clear() - out_image.copy_unpacked(src_image) - # FIXME: metadata history stuff? See misc.import_. - else: - pass - bu.cache.rollback(src_image.unpack_path) - - -# FIXME: For some reason, on this branch the parsed tree node coresponding to -# “alpine:latest” is just called “alpine”. This does not happen on the -# master branch, so I need to get to the bottom of this. -def parse_n_traverse(text): - # Parse it. - parser = lark.Lark(im.GRAMMAR_DOCKERFILE, parser="earley", - propagate_positions=True, tree_class=im.Tree) - # Avoid Lark issue #237: lark.exceptions.UnexpectedEOF if the file does not - # end in newline. - text += "\n" - try: - tree = parser.parse(text) - except lark.exceptions.UnexpectedInput as x: - ch.VERBOSE(x) # noise about what was expected in the grammar - ch.FATAL("can’t parse: %s:%d,%d\n\n%s" - % (cli.file, x.line, x.column, x.get_context(text, 39))) - ch.VERBOSE(tree.pretty()[:-1]) # rm trailing newline - - # Sometimes we exit after parsing. - if (cli.parse_only): - ch.exit(0) - - # Count the number of stages (i.e., FROM instructions) - global image_ct - image_ct = sum(1 for i in tree.children_("from_")) - - # If we use RSYNC, error out quickly if appropriate rsync(1) not present. - if (tree.child("rsync") is not None): - try: - ch.version_check(["rsync", "--version"], ch.RSYNC_MIN) - except ch.Fatal_Error: - ch.ERROR("Dockerfile uses RSYNC, so rsync(1) is required") - raise - - # Traverse the tree and do what it says. - # - # We don’t actually care whether the tree is traversed breadth-first or - # depth-first, but we *do* care that instruction nodes are visited in - # order. Neither visit() nor visit_topdown() are documented as of - # 2020-06-11 [1], but examining source code [2] shows that visit_topdown() - # uses Tree.iter_trees_topdown(), which *is* documented to be in-order [3]. - # - # This change seems to have been made in 0.8.6 (see PR #761); before then, - # visit() was in order. Therefore, we call that instead, if visit_topdown() - # is not present, to improve compatibility (see issue #792). - # - # [1]: https://lark-parser.readthedocs.io/en/latest/visitors/#visitors - # [2]: https://github.com/lark-parser/lark/blob/445c8d4/lark/visitors.py#L211 - # [3]: https://lark-parser.readthedocs.io/en/latest/classes/#tree - ml = Main_Loop() - if (hasattr(ml, 'visit_topdown')): - ml.visit_topdown(tree) - else: - ml.visit(tree) - if (ml.instruction_total_ct > 0): - if (ml.miss_ct == 0): - ml.inst_prev.checkout() - ml.inst_prev.ready() - - return ml + str(out_image.ref), "--", shell]) + # FIXME: metadata history stuff? See misc.import_. + #bu.cache.rollback(src_image.unpack_path) def modify_tree_make(src_img, cmds): + """Function that manually constructs a parse tree corresponding to a set of + “ch-image modify” commands, as though the commands had been specified in a + Dockerfile. Note that because “ch-image modify” simply executes one or + more commands inside a container, the only Dockerfile instructions we need + to consider are “FROM” and “RUN”. This results in conveniently simple + parse trees of the form: + + start + dockerfile + from_ + image_ref + IMAGE_REF + run + run_shell + LINE_CHUNK + [...] + run + run_shell + LINE_CHUNK + + e.g. for the command line + + $ ch-image modify -o foo2 -c 'echo foo' -c 'echo bar' -- foo + + this function produces the following parse tree + + start + dockerfile + from_ + image_ref + IMAGE_REF foo + run + run_shell + LINE_CHUNK echo foo + run + run_shell + LINE_CHUNK echo bar + """ # Children of dockerfile tree - #df_children = [] - src_img_ref = im.Reference(src_img) - ch.ILLERI(src_img) - #lark.Tree(lark.Token('RULE', 'image_ref')[lark.Token('IMAGE_REF', src_img_ref)]) - #df_children.append() + df_children = [] + # Metadata attribute. We use this attribute in the “_pretty” method for our + # “Tree” class. Constructing a tree without specifying a “Meta” instance that + # has been given a “line” value will result in the attribute not being present, + # which causes an error when we try to access that attribute. Here we give the + # attribute a debug value of -1 to avoid said errors. + meta = lark.tree.Meta() + meta.line = -1 + df_children.append(im.Tree(lark.Token('RULE', 'from_'), [im.Tree(lark.Token('RULE', 'image_ref'),[lark.Token('IMAGE_REF', src_img.name)], meta)], meta)) + for cmd in cmds: + df_children.append(im.Tree(lark.Token('RULE', 'run'), [im.Tree(lark.Token('RULE', 'run_shell'),[lark.Token('LINE_CHUNK', cmd)], meta)],meta)) + return im.Tree(lark.Token('RULE', 'start'), [im.Tree(lark.Token('RULE','dockerfile'), df_children)], meta) ## Supporting classes ## From d05460e7b87ad30119f2ac0bf56fcca3ad5cab32 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Tue, 19 Mar 2024 19:38:35 +0000 Subject: [PATCH 06/65] add (prototype) completion [skip ci] --- bin/ch-completion.bash | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/bin/ch-completion.bash b/bin/ch-completion.bash index 79ba59725..92d2806d8 100644 --- a/bin/ch-completion.bash +++ b/bin/ch-completion.bash @@ -259,14 +259,16 @@ _ch_convert_complete () { _image_build_opts="-b --bind --build-arg -f --file --force --force-cmd -n --dry-run --parse-only -t --tag" +_image_modify_opts="-o --out" + _image_common_opts="-a --arch --always-download --auth --break --cache --cache-large --dependencies -h --help --no-cache --no-lock --no-xattrs --profile --rebuild --password-many -q --quiet -s --storage --tls-no-verify -v --verbose --version --xattrs" -_image_subcommands="build build-cache delete gestalt - import list pull push reset undelete" +_image_subcommands="build build-cache delete gestalt import + list modify pull push reset undelete" # archs taken from ARCH_MAP in charliecloud.py _archs="amd64 arm/v5 arm/v6 arm/v7 arm64/v8 386 mips64le ppc64le s390x" @@ -374,20 +376,32 @@ _ch_image_complete () { build-cache) COMPREPLY=( $(compgen -W "--reset --gc --tree --dot" -- "$cur") ) ;; - delete|list) - if [[ "$sub_cmd" == "list" ]]; then + delete|list|modify) + case "$sub_cmd" in + list) if [[ "$prev" == "--undeletable" || "$prev" == "--undeleteable" || "$prev" == "-u" ]]; then COMPREPLY=( $(compgen -W "$(_ch_undelete_list "$strg_dir")" -- "$cur") ) return 0 fi - extras+="$extras -l --long -u --undeletable" + extras="$extras -l --long -u --undeletable" # If “cur” starts with “--undelete,” add “--undeleteable” (the less # correct version of “--undeletable”) to the list of possible # completions. if [[ ${cur::10} == "--undelete" ]]; then extras="$extras --undeleteable" fi - fi + ;; + modify) + case "$prev" in + -o|--out) + # Can’t complete for this option + COMPREPLY=() + return 0 + ;; + esac + extras="$extras $_image_modify_opts" + ;; + esac COMPREPLY=( $(compgen -W "$(_ch_list_images "$strg_dir") $extras" -- "$cur") ) __ltrim_colon_completions "$cur" ;; From 1df874f3fc33b21b9f9851885e817f7266df1330 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Thu, 28 Mar 2024 20:10:04 +0000 Subject: [PATCH 07/65] don't enforce different out image [skip ci] --- bin/ch-image.py.in | 10 ++++-- lib/build.py | 80 +++++++++++++++++++++++++--------------------- lib/image.py | 3 -- 3 files changed, 52 insertions(+), 41 deletions(-) diff --git a/bin/ch-image.py.in b/bin/ch-image.py.in index dd64a8a7c..58ec4508f 100644 --- a/bin/ch-image.py.in +++ b/bin/ch-image.py.in @@ -282,10 +282,16 @@ def main(): # modify sp = ap.add_parser("modify", "foo") add_opts(sp, build.modify, deps_check=True, stog_init=True) - sp.add_argument("-c", metavar="CMD", action="append", default=[], nargs="*", help="foo") - sp.add_argument("-o", "--out", metavar="out_image", help="foo", required=True) + sp.add_argument("-c", metavar="CMD", action="append", default=[], nargs=1, help="foo") + # FIXME: Make “--out” optional + sp.add_argument("-o", "--out", metavar="out_image", help="foo", required=False) sp.add_argument("-S", "--shell", metavar="shell", help="foo") + sp.add_argument("-u", "--unsafe", action="store_true", help="foo") sp.add_argument("image_ref", metavar="IMAGE_REF", help="image to modify") + # Using nargs="?" to make the positional argument optional (required is not a + # valid keyword for positionals). + sp.add_argument("out_image", metavar="OUT_IMAGE", help="destination of modified image", nargs="?") + #sp.add_argument("out_image", metavar="OUT_IMAGE", help="destination of modified image") # pull sp = ap.add_parser("pull", diff --git a/lib/build.py b/lib/build.py index f7ff53fc9..336b7ecc8 100644 --- a/lib/build.py +++ b/lib/build.py @@ -11,6 +11,7 @@ import shutil import subprocess import sys +import uuid import charliecloud as ch import build_cache as bu @@ -87,7 +88,6 @@ def __init__(self, *args, **kwargs): # instructions, rather than attributes of this class. def __default__(self, tree): class_ = tree.data.title() + "_G" - ch.ILLERI(tree.data.title()) if (class_ in globals()): inst = globals()[class_](tree) if (self.instruction_total_ct == 0): @@ -95,7 +95,6 @@ def __default__(self, tree): or isinstance(inst, From__G) or isinstance(inst, Instruction_No_Image))): ch.FATAL("first instruction must be ARG or FROM") - ch.ILLERI("PREV: %s" % self.inst_prev) inst.init(self.inst_prev) # The three announce_maybe() calls are clunky but I couldn’t figure # out how to avoid the repeats. @@ -306,15 +305,16 @@ def unescape(sl): return ast.literal_eval(sl) def modify(cli_): + # In this file, “cli” is used as a global variable global cli cli = cli_ + # This file assumes that global cli comes from the “build” function. If we + # don’t assign these values, it casues problems. cli.parse_only = False cli.force = ch.Force_Mode.SECCOMP cli.force_cmd = force.FORCE_CMD_DEFAULT cli.bind = [] - # FIXME: This is super kludgey - cli.tag = cli.out print(cli.image_ref) ch.ILLERI(cli.c) @@ -324,22 +324,38 @@ def modify(cli_): for c in cli.c: commands += c src_image = im.Image(im.Reference(cli.image_ref)) - out_image = cli.out + if (cli.out_image == None): + out_image = src_image + else: + out_image = im.Image(im.Reference(cli.out_image)) + ch.ILLERI("OUT_IMAGE: %s" % cli.out_image) if (not src_image.unpack_exist_p): ch.FATAL("not in storage: %s" % src_image.ref) - if (out_image == str(src_image.ref)): - ch.FATAL("output image must have different name from source (%s)" % src_image.ref) + #if (out_image == str(src_image.ref)): + # ch.FATAL("output image must have different name from source (%s)" % src_image.ref) + #if ((out_image == str(src_image.ref) or (out_image == None)) and (not cli.unsafe)): + # ch.FATAL("") + if (not cli.unsafe): + if (out_image == str(src_image.ref)): + ch.FATAL("placeholder error (src = dest)") + elif (cli.out_image == None): + ch.FATAL("placeholder error (no dest)") + + # This kludge is necessary because cli is a global variable, with cli.tag + # assumed present elsewhere in the file. cli.tag represents the image being + # built, which in our case can either be the source image or the output image + # (if specified). + cli.tag = str(out_image) + if (cli.shell is not None): shell = cli.shell else: shell = "/bin/sh" if not sys.stdin.isatty(): - # https://stackoverflow.com/a/17735803 - # commands from stdin - for line in sys.stdin: - # execute each line (analogous to RUN) - commands.append(line) + # Treat stdin as opaque blob and run that + commands = [sys.stdin] if (commands != []): + # FIXME: verify that this code path works right tree = modify_tree_make(src_image.ref, commands) # FIXME: Be more DRY in this section @@ -356,8 +372,7 @@ def modify(cli_): if (ml.instruction_total_ct > 0): if (ml.miss_ct == 0): ml.inst_prev.checkout() - ml.inst_prev.ready() - + ml.inst_prev.ready() else: # Make sure that shell exists. #try: @@ -366,13 +381,22 @@ def modify(cli_): # #print(x.__dict__) # if ("%s: No such file or directory" % shell in str(x.stderr)): # ch.FATAL("invalid shell: %s" % shell) - - out_image = im.Image(im.Reference(out_image)) - # Do something similar to “ch-image import” - out_image.unpack_clear() - out_image.copy_unpacked(src_image) + + # Generate “fake” SID + fake_sid = uuid.uuid4() + if (out_image != src_image): + #out_image = im.Image(im.Reference(out_image)) + # Do something similar to “ch-image import” + out_image.unpack_clear() + out_image.copy_unpacked(src_image) + #bu.cache.worktree_add(out_image, src_image) + bu.cache.worktree_adopt(out_image, "root") + bu.cache.ready(out_image) + bu.cache.branch_nocheckout(src_image.ref, out_image.ref) subprocess.run([ch.CH_BIN + "/ch-run", "--unsafe", "-w", str(out_image.ref), "--", shell]) + ch.VERBOSE("using SID %s" % fake_sid) + bu.cache.commit(out_image.unpack_path, fake_sid, "MODIFY interactive", []) # FIXME: metadata history stuff? See misc.import_. #bu.cache.rollback(src_image.unpack_path) @@ -381,23 +405,7 @@ def modify_tree_make(src_img, cmds): “ch-image modify” commands, as though the commands had been specified in a Dockerfile. Note that because “ch-image modify” simply executes one or more commands inside a container, the only Dockerfile instructions we need - to consider are “FROM” and “RUN”. This results in conveniently simple - parse trees of the form: - - start - dockerfile - from_ - image_ref - IMAGE_REF - run - run_shell - LINE_CHUNK - [...] - run - run_shell - LINE_CHUNK - - e.g. for the command line + to consider are “FROM” and “RUN”. E.g. for the command line $ ch-image modify -o foo2 -c 'echo foo' -c 'echo bar' -- foo diff --git a/lib/image.py b/lib/image.py index d1038fe5a..25abe4eb5 100644 --- a/lib/image.py +++ b/lib/image.py @@ -4,11 +4,9 @@ import json import os import re -import subprocess import sys import tarfile -import build_cache as bu import charliecloud as ch import filesystem as fs @@ -949,4 +947,3 @@ def terminals_cat(self, tname): """Return the concatenated values of all child terminals named tname as a string, with no delimiters. If none, return the empty string.""" return "".join(self.terminals(tname)) - From 50f917fed69e6ded84b02cd06f6b84e7c0c610ff Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Thu, 4 Apr 2024 17:13:06 +0000 Subject: [PATCH 08/65] standardize ch-run exit codes, make sure out image is different than source --- bin/ch-checkns.c | 2 +- bin/ch-image.py.in | 5 +- bin/ch-run.c | 6 +- bin/ch_core.c | 5 +- bin/ch_fuse.c | 6 +- bin/ch_misc.c | 12 ++- bin/ch_misc.h | 9 +++ doc/ch-run.rst | 19 +++-- lib/build.py | 154 +++++++++++++++--------------------- test/build/50_ch-image.bats | 30 +++++++ 10 files changed, 141 insertions(+), 107 deletions(-) diff --git a/bin/ch-checkns.c b/bin/ch-checkns.c index 10f26969a..6487ebe1f 100644 --- a/bin/ch-checkns.c +++ b/bin/ch-checkns.c @@ -71,7 +71,7 @@ void fatal_(const char *file, int line, int errno_, const char *str) char *url = "https://github.com/hpc/charliecloud/blob/master/bin/ch-checkns.c"; printf("error: %s: %d: %s\n", file, line, str); printf("errno: %d\nsee: %s\n", errno_, url); - exit(1); + exit(ERR_CHRUN); } int main(int argc, char *argv[]) diff --git a/bin/ch-image.py.in b/bin/ch-image.py.in index 58ec4508f..76156b022 100644 --- a/bin/ch-image.py.in +++ b/bin/ch-image.py.in @@ -286,11 +286,12 @@ def main(): # FIXME: Make “--out” optional sp.add_argument("-o", "--out", metavar="out_image", help="foo", required=False) sp.add_argument("-S", "--shell", metavar="shell", help="foo") - sp.add_argument("-u", "--unsafe", action="store_true", help="foo") + #sp.add_argument("-u", "--unsafe", action="store_true", help="foo") sp.add_argument("image_ref", metavar="IMAGE_REF", help="image to modify") # Using nargs="?" to make the positional argument optional (required is not a # valid keyword for positionals). - sp.add_argument("out_image", metavar="OUT_IMAGE", help="destination of modified image", nargs="?") + #sp.add_argument("out_image", metavar="OUT_IMAGE", help="destination of modified image", dest="tag") + sp.add_argument("out_image", metavar="OUT_IMAGE", help="destination of modified image") #sp.add_argument("out_image", metavar="OUT_IMAGE", help="destination of modified image") # pull diff --git a/bin/ch-run.c b/bin/ch-run.c index a2a5e8afe..5470e595b 100644 --- a/bin/ch-run.c +++ b/bin/ch-run.c @@ -445,19 +445,19 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) #ifdef HAVE_FNM_EXTMATCH exit(0); #else - exit(1); + exit(ERR_CHRUN); #endif } else if (!strcmp(arg, "seccomp")) { #ifdef HAVE_SECCOMP exit(0); #else - exit(1); + exit(ERR_CHRUN); #endif } else if (!strcmp(arg, "squash")) { #ifdef HAVE_LIBSQUASHFUSE exit(0); #else - exit(1); + exit(ERR_CHRUN); #endif } else diff --git a/bin/ch_core.c b/bin/ch_core.c index cd25e2bd0..50a85b159 100644 --- a/bin/ch_core.c +++ b/bin/ch_core.c @@ -546,7 +546,10 @@ void run_user_command(char *argv[], const char *initial_dir) if (verbose < LL_STDERR) T_ (freopen("/dev/null", "w", stderr)); execvp(argv[0], argv); // only returns if error - Tf (0, "can't execve(2): %s", argv[0]); + //Tf (0, "can't execve(2): %s", argv[0]); + //Terror (0, "can't execve(2): %s", argv[0]); + ERROR("can't execve(2): %s", argv[0]) + exit(ERR_CMD); } /* Set up the fake-syscall seccomp(2) filter. This computes and installs a diff --git a/bin/ch_fuse.c b/bin/ch_fuse.c index 0ebda0a59..2a8a56c4c 100644 --- a/bin/ch_fuse.c +++ b/bin/ch_fuse.c @@ -183,7 +183,7 @@ int sq_loop(void) // Clean up zombie child if exit signal was SIGCHLD. if (!sigchld_received) - exit_code = 0; + exit_code = 59; else { Tf (wait(&child_status) >= 0, "can't wait for child"); if (WIFEXITED(child_status)) { @@ -198,8 +198,8 @@ int sq_loop(void) // // [1]: https://codereview.stackexchange.com/a/109349 // [2]: https://man7.org/linux/man-pages/man2/wait.2.html - exit_code = 1; - VERBOSE("child terminated by signal %d", WTERMSIG(child_status)) + exit_code = 128 + WTERMSIG(child_status); + VERBOSE("child terminated by signal %d", exit_code - 128) } } diff --git a/bin/ch_misc.c b/bin/ch_misc.c index bdee7fa20..040004f9b 100644 --- a/bin/ch_misc.c +++ b/bin/ch_misc.c @@ -586,6 +586,16 @@ void msg(enum log_level level, const char *file, int line, int errno_, va_end(ap); } +void msg_error(const char *file, int line, int errno_, + const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + msgv(LL_FATAL, file, line, errno_, fmt, ap); + va_end(ap); +} + noreturn void msg_fatal(const char *file, int line, int errno_, const char *fmt, ...) { @@ -595,7 +605,7 @@ noreturn void msg_fatal(const char *file, int line, int errno_, msgv(LL_FATAL, file, line, errno_, fmt, ap); va_end(ap); - exit(EXIT_FAILURE); + exit(ERR_CHRUN); } /* va_list form of msg(). */ diff --git a/bin/ch_misc.h b/bin/ch_misc.h index f590a0890..c88103679 100644 --- a/bin/ch_misc.h +++ b/bin/ch_misc.h @@ -24,6 +24,11 @@ don’t need to worry about running out of room. */ #define WARNINGS_SIZE (4*1024) +/* Exit codes */ +#define ERR_CHRUN 57 +#define ERR_CMD 58 +#define ERR_SQUASH 59 + /* Test some value, and if it's not what we expect, exit with a fatal error. These are macros so we have access to the file and line number. @@ -62,11 +67,13 @@ #define T_(x) if (!(x)) msg_fatal(__FILE__, __LINE__, errno, NULL) #define Tf(x, ...) if (!(x)) msg_fatal(__FILE__, __LINE__, errno, __VA_ARGS__) #define Te(x, ...) if (!(x)) msg_fatal(__FILE__, __LINE__, 0, __VA_ARGS__) +#define Terror(x, ...) if (!(x)) msg_error(__FILE__, __LINE__, errno, __VA_ARGS__) #define Z_(x) if (x) msg_fatal(__FILE__, __LINE__, errno, NULL) #define Zf(x, ...) if (x) msg_fatal(__FILE__, __LINE__, errno, __VA_ARGS__) #define Ze(x, ...) if (x) msg_fatal(__FILE__, __LINE__, 0, __VA_ARGS__) #define FATAL(...) msg_fatal( __FILE__, __LINE__, 0, __VA_ARGS__); +#define ERROR(...) msg_error( __FILE__, __LINE__, 0, __VA_ARGS__); #define WARNING(...) msg(LL_WARNING, __FILE__, __LINE__, 0, __VA_ARGS__); #define INFO(...) msg(LL_INFO, __FILE__, __LINE__, 0, __VA_ARGS__); #define VERBOSE(...) msg(LL_VERBOSE, __FILE__, __LINE__, 0, __VA_ARGS__); @@ -134,6 +141,8 @@ void mkdirs(const char *base, const char *path, char **denylist, const char *scratch); void msg(enum log_level level, const char *file, int line, int errno_, const char *fmt, ...); +void msg_error(const char *file, int line, int errno_, + const char *fmt, ...); noreturn void msg_fatal(const char *file, int line, int errno_, const char *fmt, ...); bool path_exists(const char *path, struct stat *statbuf, bool follow_symlink); diff --git a/doc/ch-run.rst b/doc/ch-run.rst index 7f2d9eebf..3e4361809 100644 --- a/doc/ch-run.rst +++ b/doc/ch-run.rst @@ -742,11 +742,20 @@ would terminate the string. Exit status =========== -If there is an error during containerization, :code:`ch-run` exits with status -non-zero. If the user command is started successfully, the exit status is that -of the user command, with one exception: if the image is an internally mounted -SquashFS filesystem and the user command is killed by a signal, the exit -status is 1 regardless of the signal value. +If the user command is started successfully, the exit status is that of the user +command, with one exception: if the image is an internally mounted SquashFS +filesystem and the user command is killed by a signal, the exit status is 1 +regardless of the signal value. Alternatively, :code:`ch-run` can exit with the +following statuses: + +57 + Error during containerization (:code:`ch-run` failure) + +58 + Unable to start user command + +59 + SquashFUSE loop exited on signal before user command was complete .. include:: ./bugs.rst diff --git a/lib/build.py b/lib/build.py index 336b7ecc8..3cfc0633d 100644 --- a/lib/build.py +++ b/lib/build.py @@ -245,30 +245,7 @@ def build_arg_get(arg): ch.ERROR("Dockerfile uses RSYNC, so rsync(1) is required") raise - # Traverse the tree and do what it says. - # - # We don’t actually care whether the tree is traversed breadth-first or - # depth-first, but we *do* care that instruction nodes are visited in - # order. Neither visit() nor visit_topdown() are documented as of - # 2020-06-11 [1], but examining source code [2] shows that visit_topdown() - # uses Tree.iter_trees_topdown(), which *is* documented to be in-order [3]. - # - # This change seems to have been made in 0.8.6 (see PR #761); before then, - # visit() was in order. Therefore, we call that instead, if visit_topdown() - # is not present, to improve compatibility (see issue #792). - # - # [1]: https://lark-parser.readthedocs.io/en/latest/visitors/#visitors - # [2]: https://github.com/lark-parser/lark/blob/445c8d4/lark/visitors.py#L211 - # [3]: https://lark-parser.readthedocs.io/en/latest/classes/#tree - ml = Main_Loop() - if (hasattr(ml, 'visit_topdown')): - ml.visit_topdown(tree) - else: - ml.visit(tree) - if (ml.instruction_total_ct > 0): - if (ml.miss_ct == 0): - ml.inst_prev.checkout() - ml.inst_prev.ready() + ml = traverse_parse_tree(tree) # Check that all build arguments were consumed. if (len(cli.build_arg) != 0): @@ -291,19 +268,6 @@ def build_arg_get(arg): ## Functions ## -def unescape(sl): - # FIXME: This is also ugly and should go in the grammar. - # - # The Dockerfile spec does not precisely define string escaping, but I’m - # guessing it’s the Go rules. You will note that we are using Python rules. - # This is wrong but close enough for now (see also gripe in previous - # paragraph). - if ( not sl.startswith('"') # no start quote - and (not sl.endswith('"') or sl.endswith('\\"'))): # no end quote - sl = '"%s"' % sl - assert (len(sl) >= 2 and sl[0] == '"' and sl[-1] == '"' and sl[-2:] != '\\"') - return ast.literal_eval(sl) - def modify(cli_): # In this file, “cli” is used as a global variable global cli @@ -316,31 +280,17 @@ def modify(cli_): cli.force_cmd = force.FORCE_CMD_DEFAULT cli.bind = [] - print(cli.image_ref) - ch.ILLERI(cli.c) - ch.ILLERI(type(cli.c)) commands = [] # “Flatten” commands array for c in cli.c: commands += c src_image = im.Image(im.Reference(cli.image_ref)) - if (cli.out_image == None): - out_image = src_image - else: - out_image = im.Image(im.Reference(cli.out_image)) - ch.ILLERI("OUT_IMAGE: %s" % cli.out_image) + out_image = im.Image(im.Reference(cli.out_image)) if (not src_image.unpack_exist_p): ch.FATAL("not in storage: %s" % src_image.ref) - #if (out_image == str(src_image.ref)): - # ch.FATAL("output image must have different name from source (%s)" % src_image.ref) - #if ((out_image == str(src_image.ref) or (out_image == None)) and (not cli.unsafe)): - # ch.FATAL("") - if (not cli.unsafe): - if (out_image == str(src_image.ref)): - ch.FATAL("placeholder error (src = dest)") - elif (cli.out_image == None): - ch.FATAL("placeholder error (no dest)") - + if (cli.out_image == cli.image_ref): + ch.FATAL("output must be different from source image (%s)" % cli.image_ref) + # This kludge is necessary because cli is a global variable, with cli.tag # assumed present elsewhere in the file. cli.tag represents the image being # built, which in our case can either be the source image or the output image @@ -353,52 +303,34 @@ def modify(cli_): shell = "/bin/sh" if not sys.stdin.isatty(): # Treat stdin as opaque blob and run that - commands = [sys.stdin] + commands = [sys.stdin.read()] if (commands != []): - # FIXME: verify that this code path works right tree = modify_tree_make(src_image.ref, commands) - # FIXME: Be more DRY in this section - # Count the number of stages (i.e., FROM instructions) global image_ct image_ct = sum(1 for i in tree.children_("from_")) - ml = Main_Loop() - if (hasattr(ml, 'visit_topdown')): - ml.visit_topdown(tree) - else: - ml.visit(tree) - if (ml.instruction_total_ct > 0): - if (ml.miss_ct == 0): - ml.inst_prev.checkout() - ml.inst_prev.ready() + ml = traverse_parse_tree(tree) else: - # Make sure that shell exists. - #try: - # subprocess.run([ch.CH_BIN + "/ch-run", str(src_image.ref), "--", shell], capture_output=True).check_returncode() - #except subprocess.CalledProcessError as x: - # #print(x.__dict__) - # if ("%s: No such file or directory" % shell in str(x.stderr)): - # ch.FATAL("invalid shell: %s" % shell) - - # Generate “fake” SID + # Generate “fake” SID for build cache. We do this because we can’t compute + # an SID, but we still want to make sure that it’s unique enough that + # we’re unlikely to run into a collision. fake_sid = uuid.uuid4() - if (out_image != src_image): - #out_image = im.Image(im.Reference(out_image)) - # Do something similar to “ch-image import” - out_image.unpack_clear() - out_image.copy_unpacked(src_image) - #bu.cache.worktree_add(out_image, src_image) - bu.cache.worktree_adopt(out_image, "root") - bu.cache.ready(out_image) - bu.cache.branch_nocheckout(src_image.ref, out_image.ref) - subprocess.run([ch.CH_BIN + "/ch-run", "--unsafe", "-w", - str(out_image.ref), "--", shell]) + out_image.unpack_clear() + out_image.copy_unpacked(src_image) + bu.cache.worktree_adopt(out_image, "root") + bu.cache.ready(out_image) + bu.cache.branch_nocheckout(src_image.ref, out_image.ref) + foo = subprocess.run([ch.CH_BIN + "/ch-run", "--unsafe", "-w", + str(out_image.ref), "--", shell]) + if (foo.returncode == 58): + # FIXME: Write a better error message? + ch.FATAL("Unable to run shell: %s" % shell) + ch.ILLERI("retcode: %s" % foo.returncode) ch.VERBOSE("using SID %s" % fake_sid) bu.cache.commit(out_image.unpack_path, fake_sid, "MODIFY interactive", []) # FIXME: metadata history stuff? See misc.import_. - #bu.cache.rollback(src_image.unpack_path) def modify_tree_make(src_img, cmds): """Function that manually constructs a parse tree corresponding to a set of @@ -408,9 +340,9 @@ def modify_tree_make(src_img, cmds): to consider are “FROM” and “RUN”. E.g. for the command line $ ch-image modify -o foo2 -c 'echo foo' -c 'echo bar' -- foo - + this function produces the following parse tree - + start dockerfile from_ @@ -437,6 +369,46 @@ def modify_tree_make(src_img, cmds): df_children.append(im.Tree(lark.Token('RULE', 'run'), [im.Tree(lark.Token('RULE', 'run_shell'),[lark.Token('LINE_CHUNK', cmd)], meta)],meta)) return im.Tree(lark.Token('RULE', 'start'), [im.Tree(lark.Token('RULE','dockerfile'), df_children)], meta) +# Traverse Lark parse tree and do what it says. +# +# We don’t actually care whether the tree is traversed breadth-first or +# depth-first, but we *do* care that instruction nodes are visited in order. +# Neither visit() nor visit_topdown() are documented as of 2020-06-11 [1], but +# examining source code [2] shows that visit_topdown() uses +# Tree.iter_trees_topdown(), which *is* documented to be in-order [3]. +# +# This change seems to have been made in 0.8.6 (see PR #761); before then, +# visit() was in order. Therefore, we call that instead, if visit_topdown() is +# not present, to improve compatibility (see issue #792). +# +# [1]: https://lark-parser.readthedocs.io/en/latest/visitors/#visitors +# [2]: https://github.com/lark-parser/lark/blob/445c8d4/lark/visitors.py#L211 +# [3]: https://lark-parser.readthedocs.io/en/latest/classes/#tree +def traverse_parse_tree(tree): + ml = Main_Loop() + if (hasattr(ml, 'visit_topdown')): + ml.visit_topdown(tree) + else: + ml.visit(tree) + if (ml.instruction_total_ct > 0): + if (ml.miss_ct == 0): + ml.inst_prev.checkout() + ml.inst_prev.ready() + return ml + +def unescape(sl): + # FIXME: This is also ugly and should go in the grammar. + # + # The Dockerfile spec does not precisely define string escaping, but I’m + # guessing it’s the Go rules. You will note that we are using Python rules. + # This is wrong but close enough for now (see also gripe in previous + # paragraph). + if ( not sl.startswith('"') # no start quote + and (not sl.endswith('"') or sl.endswith('\\"'))): # no end quote + sl = '"%s"' % sl + assert (len(sl) >= 2 and sl[0] == '"' and sl[-1] == '"' and sl[-2:] != '\\"') + return ast.literal_eval(sl) + ## Supporting classes ## class Instruction(abc.ABC): diff --git a/test/build/50_ch-image.bats b/test/build/50_ch-image.bats index ab8a628f7..01657df49 100644 --- a/test/build/50_ch-image.bats +++ b/test/build/50_ch-image.bats @@ -979,3 +979,33 @@ EOF [[ $status -eq 0 ]] [[ $output = *'PWD=/bar/baz'* ]] } + +@test "ch-image modify" { + run ch-image modify -c "echo foo" -c "echo bar" -- alpine:3.17 tmpimg + echo "$output" + [[ $status -eq 0 ]] + [[ $output = *'foo'* ]] + [[ $output = *'bar'* ]] + + ch-image modify -c "touch /home/foo" -- alpine:3.17 tmpimg + run ch-run tmpimg -- ls /home + echo "$output" + [[ $status -eq 0 ]] + [[ $output = *'foo'* ]] + + printf "touch /home/bar" | ch-image modify alpine:3.17 tmpimg + run ch-run tmpimg -- ls /home + echo "$output" + [[ $status -eq 0 ]] + [[ $output = *'bar'* ]] + + run ch-image modify -c 'echo foo' -- alpine:3.17 alpine:3.17 + echo "$output" + [[ $status -eq 1 ]] + [[ $output = *'output must be different from source image'* ]] + + run ch-image modify -S foo alpine:latest tmpimg + echo "$output" + [[ $status -eq 1 ]] + [[ $output = *'Unable to run shell:'* ]] +} From 2f43bc94b766003f55e26d6c605a1f56bff46386 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Thu, 4 Apr 2024 20:28:34 +0000 Subject: [PATCH 09/65] fix a bug? --- lib/build.py | 8 ++++++-- test/build/50_ch-image.bats | 6 +++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/build.py b/lib/build.py index 3cfc0633d..a8be3a38a 100644 --- a/lib/build.py +++ b/lib/build.py @@ -286,6 +286,9 @@ def modify(cli_): commands += c src_image = im.Image(im.Reference(cli.image_ref)) out_image = im.Image(im.Reference(cli.out_image)) + ch.ILLERI("SRC REF: %s" % str(src_image.ref)) + ch.ILLERI("SRC IMAGE: %s" % str(src_image)) + ch.ILLERI("SRC REF NAME: %s" % str(src_image.ref.name)) if (not src_image.unpack_exist_p): ch.FATAL("not in storage: %s" % src_image.ref) if (cli.out_image == cli.image_ref): @@ -324,7 +327,7 @@ def modify(cli_): bu.cache.branch_nocheckout(src_image.ref, out_image.ref) foo = subprocess.run([ch.CH_BIN + "/ch-run", "--unsafe", "-w", str(out_image.ref), "--", shell]) - if (foo.returncode == 58): + if (foo.returncode == 57): # FIXME: Write a better error message? ch.FATAL("Unable to run shell: %s" % shell) ch.ILLERI("retcode: %s" % foo.returncode) @@ -364,7 +367,8 @@ def modify_tree_make(src_img, cmds): # attribute a debug value of -1 to avoid said errors. meta = lark.tree.Meta() meta.line = -1 - df_children.append(im.Tree(lark.Token('RULE', 'from_'), [im.Tree(lark.Token('RULE', 'image_ref'),[lark.Token('IMAGE_REF', src_img.name)], meta)], meta)) + #df_children.append(im.Tree(lark.Token('RULE', 'from_'), [im.Tree(lark.Token('RULE', 'image_ref'),[lark.Token('IMAGE_REF', src_img.name)], meta)], meta)) + df_children.append(im.Tree(lark.Token('RULE', 'from_'), [im.Tree(lark.Token('RULE', 'image_ref'),[lark.Token('IMAGE_REF', str(src_img))], meta)], meta)) for cmd in cmds: df_children.append(im.Tree(lark.Token('RULE', 'run'), [im.Tree(lark.Token('RULE', 'run_shell'),[lark.Token('LINE_CHUNK', cmd)], meta)],meta)) return im.Tree(lark.Token('RULE', 'start'), [im.Tree(lark.Token('RULE','dockerfile'), df_children)], meta) diff --git a/test/build/50_ch-image.bats b/test/build/50_ch-image.bats index 01657df49..e6e61ddc9 100644 --- a/test/build/50_ch-image.bats +++ b/test/build/50_ch-image.bats @@ -864,7 +864,7 @@ EOF @test 'ch-run storage errors' { run ch-run -v -w alpine:3.17 -- /bin/true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *'error: --write invalid when running by name'* ]] run ch-run -v "$CH_IMAGE_STORAGE"/img/alpine+3.17 -- /bin/true @@ -1001,11 +1001,11 @@ EOF run ch-image modify -c 'echo foo' -- alpine:3.17 alpine:3.17 echo "$output" + echo "$status" [[ $status -eq 1 ]] [[ $output = *'output must be different from source image'* ]] - run ch-image modify -S foo alpine:latest tmpimg - echo "$output" + run ch-image modify -S "foo" -- alpine:3.17 tmpimg [[ $status -eq 1 ]] [[ $output = *'Unable to run shell:'* ]] } From 491ce014c5f2a7bdd5a15017f1cf2abcab5b90d8 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Thu, 4 Apr 2024 20:38:20 +0000 Subject: [PATCH 10/65] update some return codes in CI --- test/run/ch-run_join.bats | 2 +- test/run/ch-run_misc.bats | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/run/ch-run_join.bats b/test/run/ch-run_join.bats index b87175f25..07afb2fcc 100644 --- a/test/run/ch-run_join.bats +++ b/test/run/ch-run_join.bats @@ -466,7 +466,7 @@ unset_vars () { # Can’t join namespaces of processes we don’t own. run ch-run -v --join-pid=1 "$ch_timg" -- true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *"join: can't open /proc/1/ns/user: Permission denied"* ]] # Can’t join namespaces of processes that don’t exist. diff --git a/test/run/ch-run_misc.bats b/test/run/ch-run_misc.bats index 7c65fec2d..dbe4e8c12 100644 --- a/test/run/ch-run_misc.bats +++ b/test/run/ch-run_misc.bats @@ -210,7 +210,7 @@ EOF # Error if directory does not exist. run ch-run --cd /goops "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output =~ "can't cd to /goops: No such file or directory" ]] } @@ -619,7 +619,7 @@ EOF # file does not exist run ch-run --set-env=doesnotexist.txt "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *"can't open: doesnotexist.txt: No such file or directory"* ]] # /ch/environment missing @@ -661,7 +661,7 @@ EOF # missing environment variable run ch-run --set-env='$PATH:foo' "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *'$PATH:foo: No such file or directory'* ]] } @@ -708,7 +708,7 @@ EOF printf '\n# Empty string\n\n' run ch-run --unset-env= "$ch_timg" -- env echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *'--unset-env: GLOB must have non-zero length'* ]] } From 6f8014ed1e9f2758205234303503dec2b85a7858 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Mon, 22 Apr 2024 21:33:00 +0000 Subject: [PATCH 11/65] update test suite --- bin/ch-run.c | 10 ++++----- bin/ch_core.c | 8 +++---- bin/ch_fuse.c | 2 +- bin/ch_misc.c | 4 ++-- bin/ch_misc.h | 14 ++++++------ lib/build.py | 10 ++++++--- test/build/50_ch-image.bats | 4 ++-- test/run/ch-run_escalated.bats | 6 ++--- test/run/ch-run_join.bats | 34 ++++++++++++++-------------- test/run/ch-run_misc.bats | 41 +++++++++++++++++----------------- 10 files changed, 67 insertions(+), 66 deletions(-) diff --git a/bin/ch-run.c b/bin/ch-run.c index 5470e595b..45de86fdf 100644 --- a/bin/ch-run.c +++ b/bin/ch-run.c @@ -189,7 +189,7 @@ int main(int argc, char *argv[]) if (arg_next >= argc - 1) { printf("usage: ch-run [OPTION...] IMAGE -- COMMAND [ARG...]\n"); - FATAL("IMAGE and/or COMMAND not specified"); + FATAL(0, "IMAGE and/or COMMAND not specified"); } args.c.img_ref = argv[arg_next++]; args.c.newroot = realpath_(args.c.newroot, true); @@ -210,11 +210,11 @@ int main(int argc, char *argv[]) break; case IMG_SQUASH: #ifndef HAVE_LIBSQUASHFUSE - FATAL("this ch-run does not support internal SquashFS mounts"); + FATAL(0, "this ch-run does not support internal SquashFS mounts"); #endif break; case IMG_NONE: - FATAL("unknown image type: %s", args.c.img_ref); + FATAL(0, "unknown image type: %s", args.c.img_ref); break; } @@ -461,7 +461,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) #endif } else - FATAL("unknown feature: %s", arg); + FATAL(0, "unknown feature: %s", arg); break; case -12: // --home Tf (args->c.host_home = getenv("HOME"), "--home failed: $HOME not set"); @@ -492,7 +492,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) else if (!strcmp(arg, "log-fail")) test_logging(true); else - FATAL("invalid --test argument: %s; see source code", arg); + FATAL(0, "invalid --test argument: %s; see source code", arg); break; case 'b': { // --bind char *src, *dst; diff --git a/bin/ch_core.c b/bin/ch_core.c index 50a85b159..445bd3dcd 100644 --- a/bin/ch_core.c +++ b/bin/ch_core.c @@ -218,7 +218,7 @@ void bind_mount(const char *src, const char *dst, enum bind_dep dep, if (!path_exists(dst_full, NULL, true)) switch (dep) { case BD_REQUIRED: - FATAL("can't bind: destination not found: %s", dst_full); + FATAL(0, "can't bind: destination not found: %s", dst_full); break; case BD_OPTIONAL: return; @@ -400,7 +400,7 @@ enum img_type image_type(const char *ref, const char *storage_dir) return IMG_SQUASH; // Well now we’re stumped. - FATAL("unknown image type: %s", ref); + FATAL(0, "unknown image type: %s", ref); } char *img_name2path(const char *name, const char *storage_dir) @@ -546,9 +546,7 @@ void run_user_command(char *argv[], const char *initial_dir) if (verbose < LL_STDERR) T_ (freopen("/dev/null", "w", stderr)); execvp(argv[0], argv); // only returns if error - //Tf (0, "can't execve(2): %s", argv[0]); - //Terror (0, "can't execve(2): %s", argv[0]); - ERROR("can't execve(2): %s", argv[0]) + ERROR(errno, "can't execve(2): %s", argv[0]); exit(ERR_CMD); } diff --git a/bin/ch_fuse.c b/bin/ch_fuse.c index 2a8a56c4c..b5ee928a0 100644 --- a/bin/ch_fuse.c +++ b/bin/ch_fuse.c @@ -249,7 +249,7 @@ void sq_mount(const char *img_path, char *mountpt) &OPS, sizeof(OPS), sq.ll)) { break; // success } else if (i <= 0) { - FATAL("too many FUSE errors; giving up"); + FATAL(0, "too many FUSE errors; giving up"); } else { WARNING("FUSE error mounting SquashFS; will retry"); sleep(1); diff --git a/bin/ch_misc.c b/bin/ch_misc.c index 040004f9b..674fdb2df 100644 --- a/bin/ch_misc.c +++ b/bin/ch_misc.c @@ -449,7 +449,7 @@ void test_logging(bool fail) { INFO("info"); WARNING("warning"); if (fail) - FATAL("the program failed inexplicably (\"log-fail\" specified)"); + FATAL(0, "the program failed inexplicably (\"log-fail\" specified)"); exit(0); } @@ -587,7 +587,7 @@ void msg(enum log_level level, const char *file, int line, int errno_, } void msg_error(const char *file, int line, int errno_, - const char *fmt, ...) + const char *fmt, ...) { va_list ap; diff --git a/bin/ch_misc.h b/bin/ch_misc.h index c88103679..c86755236 100644 --- a/bin/ch_misc.h +++ b/bin/ch_misc.h @@ -72,13 +72,13 @@ #define Zf(x, ...) if (x) msg_fatal(__FILE__, __LINE__, errno, __VA_ARGS__) #define Ze(x, ...) if (x) msg_fatal(__FILE__, __LINE__, 0, __VA_ARGS__) -#define FATAL(...) msg_fatal( __FILE__, __LINE__, 0, __VA_ARGS__); -#define ERROR(...) msg_error( __FILE__, __LINE__, 0, __VA_ARGS__); -#define WARNING(...) msg(LL_WARNING, __FILE__, __LINE__, 0, __VA_ARGS__); -#define INFO(...) msg(LL_INFO, __FILE__, __LINE__, 0, __VA_ARGS__); -#define VERBOSE(...) msg(LL_VERBOSE, __FILE__, __LINE__, 0, __VA_ARGS__); -#define DEBUG(...) msg(LL_DEBUG, __FILE__, __LINE__, 0, __VA_ARGS__); -#define TRACE(...) msg(LL_TRACE, __FILE__, __LINE__, 0, __VA_ARGS__); +#define FATAL(e, ...) msg_fatal( __FILE__, __LINE__, e, __VA_ARGS__); +#define ERROR(e, ...) msg_error( __FILE__, __LINE__, e, __VA_ARGS__); +#define WARNING(...) msg(LL_WARNING, __FILE__, __LINE__, 0, __VA_ARGS__); +#define INFO(...) msg(LL_INFO, __FILE__, __LINE__, 0, __VA_ARGS__); +#define VERBOSE(...) msg(LL_VERBOSE, __FILE__, __LINE__, 0, __VA_ARGS__); +#define DEBUG(...) msg(LL_DEBUG, __FILE__, __LINE__, 0, __VA_ARGS__); +#define TRACE(...) msg(LL_TRACE, __FILE__, __LINE__, 0, __VA_ARGS__); /** Types **/ diff --git a/lib/build.py b/lib/build.py index a8be3a38a..dc3b5389d 100644 --- a/lib/build.py +++ b/lib/build.py @@ -322,18 +322,22 @@ def modify(cli_): fake_sid = uuid.uuid4() out_image.unpack_clear() out_image.copy_unpacked(src_image) - bu.cache.worktree_adopt(out_image, "root") + bu.cache.worktree_adopt(out_image, src_image.ref.for_path) bu.cache.ready(out_image) bu.cache.branch_nocheckout(src_image.ref, out_image.ref) foo = subprocess.run([ch.CH_BIN + "/ch-run", "--unsafe", "-w", str(out_image.ref), "--", shell]) - if (foo.returncode == 57): + if (foo.returncode == 58): # FIXME: Write a better error message? ch.FATAL("Unable to run shell: %s" % shell) ch.ILLERI("retcode: %s" % foo.returncode) ch.VERBOSE("using SID %s" % fake_sid) - bu.cache.commit(out_image.unpack_path, fake_sid, "MODIFY interactive", []) # FIXME: metadata history stuff? See misc.import_. + if (out_image.metadata["history"] == []): + out_image.metadata["history"].append({ "empty_layer": False, + "command": "ch-image import"}) + out_image.metadata_save() + bu.cache.commit(out_image.unpack_path, fake_sid, "MODIFY interactive", []) def modify_tree_make(src_img, cmds): """Function that manually constructs a parse tree corresponding to a set of diff --git a/test/build/50_ch-image.bats b/test/build/50_ch-image.bats index e6e61ddc9..f594e606c 100644 --- a/test/build/50_ch-image.bats +++ b/test/build/50_ch-image.bats @@ -869,12 +869,12 @@ EOF run ch-run -v "$CH_IMAGE_STORAGE"/img/alpine+3.17 -- /bin/true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *"error: can't run directory images from storage (hint: run by name)"* ]] run ch-run -v -s /doesnotexist alpine:3.17 -- /bin/true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *'warning: storage directory not found: /doesnotexist'* ]] [[ $output = *"error: can't stat: alpine:3.17: No such file or directory"* ]] } diff --git a/test/run/ch-run_escalated.bats b/test/run/ch-run_escalated.bats index 9bac753f6..e90492ed8 100644 --- a/test/run/ch-run_escalated.bats +++ b/test/run/ch-run_escalated.bats @@ -15,7 +15,7 @@ load ../common [[ -g $ch_run_tmp ]] run "$ch_run_tmp" --version echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *': please report this bug ('* ]] rm "$ch_run_tmp" } @@ -32,7 +32,7 @@ load ../common [[ -u $ch_run_tmp ]] run "$ch_run_tmp" --version echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *': please report this bug ('* ]] sudo rm "$ch_run_tmp" } @@ -71,7 +71,7 @@ load ../common fi run sudo -u root -g "$(id -gn)" "$ch_runfile" -v --version echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *'please report this bug ('* ]] } diff --git a/test/run/ch-run_join.bats b/test/run/ch-run_join.bats index 07afb2fcc..3a7555686 100644 --- a/test/run/ch-run_join.bats +++ b/test/run/ch-run_join.bats @@ -267,36 +267,36 @@ unset_vars () { # --join but no join count run ch-run --join "$ch_timg" -- true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output =~ 'join: no valid peer group size found' ]] ipc_clean_p # join count no digits run ch-run --join-ct=a "$ch_timg" -- true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output =~ 'join-ct: no digits found' ]] SLURM_CPUS_ON_NODE=a run ch-run --join "$ch_timg" -- true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output =~ 'SLURM_CPUS_ON_NODE: no digits found' ]] ipc_clean_p # join count empty string run ch-run --join-ct='' "$ch_timg" -- true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output =~ '--join-ct: no digits found' ]] SLURM_CPUS_ON_NODE=-1 run ch-run --join "$ch_timg" -- true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output =~ 'join: no valid peer group size found' ]] ipc_clean_p # --join-ct digits followed by extra goo (OK from environment variable) run ch-run --join-ct=1a "$ch_timg" -- true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output =~ '--join-ct: extra characters after digits' ]] ipc_clean_p @@ -306,48 +306,48 @@ unset_vars () { # join count above INT_MAX run ch-run --join-ct=2147483648 "$ch_timg" -- true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output =~ $range_re ]] SLURM_CPUS_ON_NODE=2147483648 \ run ch-run --join "$ch_timg" -- true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output =~ $range_re ]] ipc_clean_p # join count below INT_MIN run ch-run --join-ct=-2147483649 "$ch_timg" -- true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output =~ $range_re ]] SLURM_CPUS_ON_NODE=-2147483649 \ run ch-run --join "$ch_timg" -- true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output =~ $range_re ]] ipc_clean_p # join count above LONG_MAX run ch-run --join-ct=9223372036854775808 "$ch_timg" -- true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output =~ $range_re ]] SLURM_CPUS_ON_NODE=9223372036854775808 \ run ch-run --join "$ch_timg" -- true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output =~ $range_re ]] ipc_clean_p # join count below LONG_MIN run ch-run --join-ct=-9223372036854775809 "$ch_timg" -- true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output =~ $range_re ]] SLURM_CPUS_ON_NODE=-9223372036854775809 \ run ch-run --join "$ch_timg" -- true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output =~ $range_re ]] ipc_clean_p } @@ -361,11 +361,11 @@ unset_vars () { # join tag empty string run ch-run --join-tag='' "$ch_timg" -- true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output =~ 'join: peer group tag cannot be empty string' ]] SLURM_STEP_ID='' run ch-run --join "$ch_timg" -- true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output =~ 'join: peer group tag cannot be empty string' ]] ipc_clean_p } @@ -473,7 +473,7 @@ unset_vars () { pid=2147483647 run ch-run -v --join-pid="$pid" "$ch_timg" -- true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *"join: no PID ${pid}: /proc/${pid}/ns/user not found"* ]] } diff --git a/test/run/ch-run_misc.bats b/test/run/ch-run_misc.bats index dbe4e8c12..3f88bbef0 100644 --- a/test/run/ch-run_misc.bats +++ b/test/run/ch-run_misc.bats @@ -121,7 +121,7 @@ EOF run ch-run --home "$ch_timg" -- /bin/sh -c 'echo $HOME' export HOME="$home_tmp" echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] # shellcheck disable=SC2016 [[ $output = *'--home failed: $HOME not set'* ]] @@ -132,7 +132,7 @@ EOF run ch-run --home "$ch_timg" -- /bin/sh -c 'echo $HOME' export USER=$user_tmp echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] # shellcheck disable=SC2016 [[ $output = *'$USER not set'* ]] } @@ -291,10 +291,10 @@ EOF [[ $status -eq 0 ]] # --home - run ch-run --home "$img" -- ls -lah /home + run ch-run --home "$img" -- ls -lAh /home echo "$output" [[ $status -eq 0 ]] - [[ $(echo "$output" | wc -l) -eq 3 ]] + [[ $(echo "$output" | wc -l) -eq 5 ]] [[ $output = *directory-in-home* ]] [[ $output = *file-in-home* ]] [[ $output = *"$USER"* ]] @@ -625,7 +625,7 @@ EOF # /ch/environment missing run ch-run --set-env "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *"can't open: /ch/environment: No such file or directory"* ]] # Note: I’m not sure how to test an error during reading, i.e., getline(3) @@ -635,14 +635,14 @@ EOF echo 'FOO bar' > "$f_in" run ch-run --set-env="$f_in" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *"can't parse variable: no delimiter: ${f_in}:1"* ]] # invalid line: no name echo '=bar' > "$f_in" run ch-run --set-env="$f_in" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *"can't parse variable: empty name: ${f_in}:1"* ]] } @@ -947,7 +947,7 @@ EOF # This should start up the container OK but fail to find the user command. run ch-run "$img" -- /bin/true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 58 ]] [[ $output = *"can't execve(2): /bin/true: No such file or directory"* ]] # For each required file, we want a correct error if it’s missing. @@ -958,7 +958,7 @@ EOF run ch-run "$img" -- /bin/true touch "${img}/${f}" # restore before test fails for idempotency echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] r="can't bind: destination not found: .+/${f}" echo "expected: ${r}" [[ $output =~ $r ]] @@ -971,7 +971,7 @@ EOF run ch-run "$img" -- /bin/true touch "${img}/${f}" # restore before test fails for idempotency echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 58 ]] [[ $output = *"can't execve(2): /bin/true: No such file or directory"* ]] done @@ -984,7 +984,7 @@ EOF rmdir "${img}/${f}" # restore before test fails for idempotency touch "${img}/${f}" echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] r="can't bind .+ to /.+/${f}: Not a directory" echo "expected: ${r}" [[ $output =~ $r ]] @@ -997,7 +997,7 @@ EOF run ch-run "$img" -- /bin/true mkdir "${img}/${d}" # restore before test fails for idempotency echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] r="can't bind: destination not found: .+/${d}" echo "expected: ${r}" [[ $output =~ $r ]] @@ -1012,7 +1012,7 @@ EOF rm "${img}/${d}" # restore before test fails for idempotency mkdir "${img}/${d}" echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] r="can't bind .+ to /.+/${d}: Not a directory" echo "expected: ${r}" [[ $output =~ $r ]] @@ -1023,7 +1023,7 @@ EOF run ch-run --private-tmp "$img" -- /bin/true mkdir "${img}/tmp" # restore before test fails for idempotency echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] r="can't mount tmpfs at /.+/tmp: No such file or directory" echo "expected: ${r}" [[ $output =~ $r ]] @@ -1033,13 +1033,13 @@ EOF run ch-run "$img" -- /bin/true mkdir "${img}/home" # restore before test fails for idempotency echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 58 ]] [[ $output = *"can't execve(2): /bin/true: No such file or directory"* ]] # Everything should be restored and back to the original error. run ch-run "$img" -- /bin/true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 58 ]] [[ $output = *"can't execve(2): /bin/true: No such file or directory"* ]] # At this point, there should be exactly two each of passwd and group @@ -1143,7 +1143,7 @@ EOF # subprocess failure at quiet level 2 run ch-run -qq "$ch_timg" -- doesnotexist echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 58 ]] [[ $output = *"error: can't execve(2): doesnotexist: No such file or directory"* ]] # quiet level 3 @@ -1156,13 +1156,12 @@ EOF # subprocess failure at quiet level 3 run ch-run -qqq "$ch_timg" -- doesnotexist echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 58 ]] [[ $output != *"error: can't execve(2): doesnotexist: No such file or directory"* ]] # failure at quiet level 3 run ch-run -qqq --test=log-fail - echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output != *'info'* ]] [[ $output != *'warning: warning'* ]] [[ $output = *'error: the program failed inexplicably'* ]] @@ -1174,6 +1173,6 @@ EOF # bad tmpfs size run ch-run --write-fake=foo "$ch_timg" -- true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output == *'cannot mount tmpfs for overlay: Invalid argument'* ]] } From 4996b4e9b65164e7d3a252030b086abf8bfa01e1 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Mon, 22 Apr 2024 22:19:31 +0000 Subject: [PATCH 12/65] update more exit statuses --- examples/multistage/test.bats | 2 +- test/build/50_ch-image.bats | 2 ++ test/run/ch-run_misc.bats | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/examples/multistage/test.bats b/examples/multistage/test.bats index ae0dcaa45..2a48f3e7a 100644 --- a/examples/multistage/test.bats +++ b/examples/multistage/test.bats @@ -37,7 +37,7 @@ setup () { # Can’t run GCC. run ch-run "$ch_img" -- gcc --version echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 58 ]] [[ $output = *'gcc: No such file or directory'* ]] # No GCC or Make. diff --git a/test/build/50_ch-image.bats b/test/build/50_ch-image.bats index f594e606c..b6b71ee60 100644 --- a/test/build/50_ch-image.bats +++ b/test/build/50_ch-image.bats @@ -981,6 +981,8 @@ EOF } @test "ch-image modify" { + ch-image reset + run ch-image modify -c "echo foo" -c "echo bar" -- alpine:3.17 tmpimg echo "$output" [[ $status -eq 0 ]] diff --git a/test/run/ch-run_misc.bats b/test/run/ch-run_misc.bats index 3f88bbef0..2a298f6f7 100644 --- a/test/run/ch-run_misc.bats +++ b/test/run/ch-run_misc.bats @@ -317,7 +317,7 @@ EOF # empty argument to --bind run ch-run -b '' "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 64 ]] [[ $output = *'--bind: no source provided'* ]] # source not provided @@ -911,7 +911,7 @@ EOF # image is file but not sqfs run ch-run -vv ./fixtures/README -- /bin/true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *'magic expected: 6873 7173; actual: 596f 7520'* ]] [[ $output = *'unknown image type: '*'/fixtures/README'* ]] From e641935f33a0200319fd49880e99eb30f3662254 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Tue, 23 Apr 2024 19:55:20 +0000 Subject: [PATCH 13/65] wtf --- test/run/ch-run_misc.bats | 45 +++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/test/run/ch-run_misc.bats b/test/run/ch-run_misc.bats index 2a298f6f7..f227bc426 100644 --- a/test/run/ch-run_misc.bats +++ b/test/run/ch-run_misc.bats @@ -311,118 +311,127 @@ EOF # no argument to --bind run ch-run "$ch_timg" -b echo "$output" + echo "STATUS" + echo "$status" + echo "STATUS" [[ $status -eq 64 ]] [[ $output = *'option requires an argument'* ]] # empty argument to --bind run ch-run -b '' "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 64 ]] + echo "STATUS" + echo "$status" + echo "STATUS" + [[ $status -eq 57 ]] [[ $output = *'--bind: no source provided'* ]] # source not provided run ch-run -b :/mnt/9 "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 1 ]] + echo "STATUS" + echo "$status" + echo "STATUS" + [[ $status -eq 57 ]] [[ $output = *'--bind: no source provided'* ]] # destination not provided run ch-run -b "${bind1_dir}:" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *'--bind: no destination provided'* ]] # destination is / run ch-run -b "${bind1_dir}:/" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *"--bind: destination can't be /"* ]] # destination is relative run ch-run -b "${bind1_dir}:foo" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *"--bind: destination must be absolute"* ]] # destination climbs out of image, exists run ch-run -b "${bind1_dir}:/.." "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *"can't bind: "*"/${USER}.ch not subdirectory of "*"/${USER}.ch/mnt"* ]] # destination climbs out of image, does not exist run ch-run -b "${bind1_dir}:/../doesnotexist/a" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *"can't mkdir: "*"/${USER}.ch/doesnotexist not subdirectory of "*"/${USER}.ch/mnt"* ]] [[ ! -e ${ch_imgdir}/doesnotexist ]] # source does not exist run ch-run -b "/doesnotexist" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *"can't bind: source not found: /doesnotexist"* ]] # destination does not exist and image is not writeable run ch-run -b "${bind1_dir}:/doesnotexist" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *"can't mkdir: "*"/${USER}.ch/mnt/doesnotexist: Read-only file system"* ]] # neither source nor destination exist run ch-run -b /doesnotexist-out:/doesnotexist-in "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *"can't bind: source not found: /doesnotexist-out"* ]] # correct bind followed by source does not exist run ch-run -b "${bind1_dir}:/mnt/0" -b /doesnotexist "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *"can't bind: source not found: /doesnotexist"* ]] # correct bind followed by destination does not exist run ch-run -b "${bind1_dir}:/mnt/0" -b "${bind2_dir}:/doesnotexist" \ "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *"can't mkdir: "*"/${USER}.ch/mnt/doesnotexist: Read-only file system"* ]] # destination is broken symlink run ch-run -b "${bind1_dir}:/mnt/link-b0rken-abs" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *"can't mkdir: symlink not relative: "*"/${USER}.ch/mnt/mnt/link-b0rken-abs"* ]] # destination is absolute symlink outside image run ch-run -b "${bind1_dir}:/mnt/link-bad-abs" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *"can't bind: "*" not subdirectory of"* ]] # destination relative symlink outside image run ch-run -b "${bind1_dir}:/mnt/link-bad-rel" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *"can't bind: "*" not subdirectory of"* ]] # mkdir(2) under existing bind-mount, default, first level run ch-run -b "${bind1_dir}:/proc/doesnotexist" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *"can't mkdir: "*"/${USER}.ch/mnt/proc/doesnotexist under existing bind-mount "*"/${USER}.ch/mnt/proc "* ]] # mkdir(2) under existing bind-mount, user-supplied, first level run ch-run -b "${bind1_dir}:/mnt/0" \ -b "${bind2_dir}:/mnt/0/foo" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *"can't mkdir: "*"/${USER}.ch/mnt/mnt/0/foo under existing bind-mount "*"/${USER}.ch/mnt/mnt/0 "* ]] # mkdir(2) under existing bind-mount, default, 2nd level run ch-run -b "${bind1_dir}:/proc/sys/doesnotexist" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 1 ]] + [[ $status -eq 57 ]] [[ $output = *"can't mkdir: "*"/${USER}.ch/mnt/proc/sys/doesnotexist under existing bind-mount "*"/${USER}.ch/mnt/proc "* ]] } From 151592f7cd1229405355553d504f328371fc1254 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Wed, 24 Apr 2024 16:26:17 +0000 Subject: [PATCH 14/65] enable CI ssh --- .github/workflows/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5cc5e943c..f264b955d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -51,10 +51,10 @@ jobs: # only happen on CI. Comment out unless needed. WARNING: tmate.io has # access to unencrypted SSH traffic. # See: https://github.com/marketplace/actions/debugging-with-tmate - #- name: set up tmate session - # uses: mxschmitt/action-tmate@v3 - # with: - # detached: true + - name: set up tmate session + uses: mxschmitt/action-tmate@v3 + with: + detached: true - name: early setup & validation run: | From e0bc574bed92c3d859cf0b5a25f37eb5a572c3ed Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Wed, 24 Apr 2024 20:17:21 +0000 Subject: [PATCH 15/65] run ci From 7d6f66ec7aca8b50f808430a4fc738f3bd440da7 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Wed, 24 Apr 2024 21:43:42 +0000 Subject: [PATCH 16/65] oops --- test/build/50_ch-image.bats | 1 - 1 file changed, 1 deletion(-) diff --git a/test/build/50_ch-image.bats b/test/build/50_ch-image.bats index b6b71ee60..014aa90a0 100644 --- a/test/build/50_ch-image.bats +++ b/test/build/50_ch-image.bats @@ -981,7 +981,6 @@ EOF } @test "ch-image modify" { - ch-image reset run ch-image modify -c "echo foo" -c "echo bar" -- alpine:3.17 tmpimg echo "$output" From 2c8b2f720e52a594f2f9bb5c33d91b81d8c31bef Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Thu, 25 Apr 2024 15:52:02 +0000 Subject: [PATCH 17/65] CI debugging --- lib/build.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/build.py b/lib/build.py index dc3b5389d..12c403816 100644 --- a/lib/build.py +++ b/lib/build.py @@ -284,6 +284,7 @@ def modify(cli_): # “Flatten” commands array for c in cli.c: commands += c + ch.ILLERI("COMMANDS: %s" % commands) src_image = im.Image(im.Reference(cli.image_ref)) out_image = im.Image(im.Reference(cli.out_image)) ch.ILLERI("SRC REF: %s" % str(src_image.ref)) @@ -310,6 +311,8 @@ def modify(cli_): if (commands != []): tree = modify_tree_make(src_image.ref, commands) + ch.ILLERI("TREE") + ch.ILLERI(tree) # Count the number of stages (i.e., FROM instructions) global image_ct image_ct = sum(1 for i in tree.children_("from_")) From 731cfe859491833f5d5e79cd2b5dcffbe97ed7c6 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Thu, 25 Apr 2024 16:03:40 +0000 Subject: [PATCH 18/65] CI debugging again --- lib/build.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/build.py b/lib/build.py index 12c403816..3c8cbaa94 100644 --- a/lib/build.py +++ b/lib/build.py @@ -309,6 +309,7 @@ def modify(cli_): # Treat stdin as opaque blob and run that commands = [sys.stdin.read()] if (commands != []): + ch.ILLERI("commands (pre tree): %s" % commands) tree = modify_tree_make(src_image.ref, commands) ch.ILLERI("TREE") @@ -365,6 +366,8 @@ def modify_tree_make(src_img, cmds): run_shell LINE_CHUNK echo bar """ + ch.ILLERI("commands (in tree func): %s" % cmds) + # Children of dockerfile tree df_children = [] # Metadata attribute. We use this attribute in the “_pretty” method for our @@ -376,7 +379,9 @@ def modify_tree_make(src_img, cmds): meta.line = -1 #df_children.append(im.Tree(lark.Token('RULE', 'from_'), [im.Tree(lark.Token('RULE', 'image_ref'),[lark.Token('IMAGE_REF', src_img.name)], meta)], meta)) df_children.append(im.Tree(lark.Token('RULE', 'from_'), [im.Tree(lark.Token('RULE', 'image_ref'),[lark.Token('IMAGE_REF', str(src_img))], meta)], meta)) + ch.ILLERI("ADDING COMMANDS TO TREE") for cmd in cmds: + ch.ILLERI("command: %s" % cmd) df_children.append(im.Tree(lark.Token('RULE', 'run'), [im.Tree(lark.Token('RULE', 'run_shell'),[lark.Token('LINE_CHUNK', cmd)], meta)],meta)) return im.Tree(lark.Token('RULE', 'start'), [im.Tree(lark.Token('RULE','dockerfile'), df_children)], meta) From fd45f6465a8dff671b0d8f6452037ab962b05fb1 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Thu, 25 Apr 2024 16:14:57 +0000 Subject: [PATCH 19/65] CI debugging again pt 2 --- lib/build.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/build.py b/lib/build.py index 3c8cbaa94..2825cb722 100644 --- a/lib/build.py +++ b/lib/build.py @@ -307,6 +307,7 @@ def modify(cli_): shell = "/bin/sh" if not sys.stdin.isatty(): # Treat stdin as opaque blob and run that + ch.ILLERI("FOUND IT!!!") commands = [sys.stdin.read()] if (commands != []): ch.ILLERI("commands (pre tree): %s" % commands) From 05dac9b950d29c9a016ab899e28e16f063d9a407 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Thu, 25 Apr 2024 16:46:16 +0000 Subject: [PATCH 20/65] fix a CI bug --- lib/build.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/lib/build.py b/lib/build.py index 2825cb722..f5645daa6 100644 --- a/lib/build.py +++ b/lib/build.py @@ -284,12 +284,8 @@ def modify(cli_): # “Flatten” commands array for c in cli.c: commands += c - ch.ILLERI("COMMANDS: %s" % commands) src_image = im.Image(im.Reference(cli.image_ref)) out_image = im.Image(im.Reference(cli.out_image)) - ch.ILLERI("SRC REF: %s" % str(src_image.ref)) - ch.ILLERI("SRC IMAGE: %s" % str(src_image)) - ch.ILLERI("SRC REF NAME: %s" % str(src_image.ref.name)) if (not src_image.unpack_exist_p): ch.FATAL("not in storage: %s" % src_image.ref) if (cli.out_image == cli.image_ref): @@ -305,16 +301,17 @@ def modify(cli_): shell = cli.shell else: shell = "/bin/sh" - if not sys.stdin.isatty(): + # Second condition here is to ensure that “commands” does’t get overwritten + # in the case where “ch-image modify” isn’t called from a terminal session + # (e.g. in Github actions). I’m considering this a temporary fix, although I + # think a case could also be made for “-c” to have precedence over pipeline + # input. + if ((not sys.stdin.isatty()) and (commands == [])): # Treat stdin as opaque blob and run that - ch.ILLERI("FOUND IT!!!") commands = [sys.stdin.read()] if (commands != []): - ch.ILLERI("commands (pre tree): %s" % commands) tree = modify_tree_make(src_image.ref, commands) - ch.ILLERI("TREE") - ch.ILLERI(tree) # Count the number of stages (i.e., FROM instructions) global image_ct image_ct = sum(1 for i in tree.children_("from_")) From 00e2ec6af11c9d3c28d1d8596179c923c8929895 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Thu, 25 Apr 2024 17:10:43 +0000 Subject: [PATCH 21/65] more ci --- lib/build.py | 2 -- test/build/50_ch-image.bats | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/build.py b/lib/build.py index f5645daa6..2c71d91c7 100644 --- a/lib/build.py +++ b/lib/build.py @@ -364,8 +364,6 @@ def modify_tree_make(src_img, cmds): run_shell LINE_CHUNK echo bar """ - ch.ILLERI("commands (in tree func): %s" % cmds) - # Children of dockerfile tree df_children = [] # Metadata attribute. We use this attribute in the “_pretty” method for our diff --git a/test/build/50_ch-image.bats b/test/build/50_ch-image.bats index 014aa90a0..347aa9f03 100644 --- a/test/build/50_ch-image.bats +++ b/test/build/50_ch-image.bats @@ -1002,11 +1002,11 @@ EOF run ch-image modify -c 'echo foo' -- alpine:3.17 alpine:3.17 echo "$output" - echo "$status" [[ $status -eq 1 ]] [[ $output = *'output must be different from source image'* ]] - run ch-image modify -S "foo" -- alpine:3.17 tmpimg + run ch-image modify -S "doesnotexist" -- alpine:3.17 tmpimg + echo "$output" [[ $status -eq 1 ]] [[ $output = *'Unable to run shell:'* ]] } From de7d63cae78ec734805e0420e104e14053e20c94 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Thu, 25 Apr 2024 17:36:49 +0000 Subject: [PATCH 22/65] uggh --- lib/build.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/build.py b/lib/build.py index 2c71d91c7..96f728942 100644 --- a/lib/build.py +++ b/lib/build.py @@ -297,10 +297,12 @@ def modify(cli_): # (if specified). cli.tag = str(out_image) + ch.ILLERI("CLI SHELL: %s" % cli.shell) if (cli.shell is not None): shell = cli.shell else: shell = "/bin/sh" + ch.ILLERI("SHELL: %s" % shell) # Second condition here is to ensure that “commands” does’t get overwritten # in the case where “ch-image modify” isn’t called from a terminal session # (e.g. in Github actions). I’m considering this a temporary fix, although I From f7b87e514c392d2a44d402ec3bc1da9f428438ce Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Thu, 25 Apr 2024 17:41:11 +0000 Subject: [PATCH 23/65] lets try this --- lib/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/build.py b/lib/build.py index 96f728942..9bac52345 100644 --- a/lib/build.py +++ b/lib/build.py @@ -311,7 +311,7 @@ def modify(cli_): if ((not sys.stdin.isatty()) and (commands == [])): # Treat stdin as opaque blob and run that commands = [sys.stdin.read()] - if (commands != []): + if (commands not in [[],['']]): tree = modify_tree_make(src_image.ref, commands) # Count the number of stages (i.e., FROM instructions) From 4b84644aa4b28c168dd0c2d9172a9f318adfe74b Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Thu, 25 Apr 2024 19:22:32 +0000 Subject: [PATCH 24/65] fix disabled cache bug --- lib/build.py | 2 ++ lib/build_cache.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/lib/build.py b/lib/build.py index 9bac52345..af5e45bc7 100644 --- a/lib/build.py +++ b/lib/build.py @@ -311,6 +311,8 @@ def modify(cli_): if ((not sys.stdin.isatty()) and (commands == [])): # Treat stdin as opaque blob and run that commands = [sys.stdin.read()] + # Again, the fact that we have to check against both “[]” and “['']” is to + # get CI to work... if (commands not in [[],['']]): tree = modify_tree_make(src_image.ref, commands) diff --git a/lib/build_cache.py b/lib/build_cache.py index 41ecded9d..92e1ba3a5 100644 --- a/lib/build_cache.py +++ b/lib/build_cache.py @@ -1379,6 +1379,9 @@ def __init__(self, *args): def __str__(self): return "disabled" + def branch_nocheckout(self, src_ref, dest): + pass + def checkout(self, image, git_hash, base_image): ch.INFO("copying image ...") image.unpack_clear() From 1afc853ad10f3f9ad4b8bb3db9e18e9a81f1f790 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Fri, 26 Apr 2024 14:37:18 +0000 Subject: [PATCH 25/65] update docs [skip ci] --- doc/ch-image.rst | 41 ++++++++++++++--------------------------- 1 file changed, 14 insertions(+), 27 deletions(-) diff --git a/doc/ch-image.rst b/doc/ch-image.rst index 3cd330554..71c84f2db 100644 --- a/doc/ch-image.rst +++ b/doc/ch-image.rst @@ -1905,37 +1905,25 @@ Synopsis :: - $ ch-image [...] modify [...] TARGET + $ ch-image [...] modify [...] TARGET DEST Description ----------- -This subcommand starts a shell on the image named :code:`TARGET`, in order to -edit the image interactively. It is similar to a :code:`RUN` instruction that -starts an interactive shell. By default, ask the user whether to save changes -when the shell exits. +This subcommand makes a copy of image :code:`TARGET`, named :code:`DEST`, and +starts a shell on :code:`DEST` in order to edit the image interactively. It is +similar to a :code:`RUN` instruction that starts an interactive shell. If there +is already an image named :code:`DEST`, it will be overwritten. :code:`DEST` must +be different than :code:`TARGET`. -Options: - - :code:`-m MSG` - Use :code:`MSG` to identify the edits to the build cache. That is, if you - run this command twice with the same :code:`TARGET`, the same :code:`-o - DEST`, and the same :code:`MSG`, the second session will overwrite the - first. (Without :code:`-o`, the second session will build atop the first.) - By default, every interactive session is considered different from every - other, as if a random :code:`MSG` were entered. - - :code:`-o`, :code:`--out DEST` - Save the results in image named :code:`DEST`, leaving :code:`TARGET` - unchanged. - - :code:`-s`, :code:`--shell SHELL` - Start :code:`SHELL` instead of :code:`/bin/sh`. + :code:`-c CMD` + Run :code:`CMD` inside the container, as though specified by the :code:`RUN` + instruction in a Dockerfile. Can be repeated to run multiple commands + sequentially. - :code:`-y`, :code:`--yes` - Do not prompt the user to save. Instead, save if the shell exits - successfully, and roll back if it exits unsuccessfully, e.g. by executing - :code:`exit 1`. + :code:`-S SHELL` + Use shell :code:`SHELL` for interactive session, rather than the default + :code:`/bin/sh`. .. warning:: @@ -1950,13 +1938,12 @@ Examples To edit the image :code:`foo`, adding :code:`/opt/lib` to the default shared library search path, producing image :code:`bar` as the result:: - $ ch-image modify -o bar foo + $ ch-image modify bar foo [...] > emacs /etc/ld.so.conf [... append line “/opt/lib” to the file ...] > ldconfig > exit - Save changes ([y]/n)? y committing ... [...] From c6ae8a965e5b72288514f4a6ecc0d4d7da254950 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Mon, 29 Apr 2024 20:55:21 +0000 Subject: [PATCH 26/65] weird docker message? --- test/build/50_dockerfile.bats | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/build/50_dockerfile.bats b/test/build/50_dockerfile.bats index e25f99d59..dcf6e5be8 100644 --- a/test/build/50_dockerfile.bats +++ b/test/build/50_dockerfile.bats @@ -1160,7 +1160,8 @@ EOF [[ $status -ne 0 ]] [[ $output = *'error: no context because '?'-'?' given'* \ || $output = *'COPY failed: file not found in build context or'* \ - || $output = *'no such file or directory'* ]] + || $output = *'no such file or directory'* \ + || $output = *'failed to compute cache key: failed to calculate checksum of ref'* ]] } From 4e61c0657064c2cf5296173f270e7c929a0da8a9 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Tue, 30 Apr 2024 15:40:04 +0000 Subject: [PATCH 27/65] some cleanup [skip ci] --- .github/workflows/main.yml | 8 ++++---- bin/ch-completion.bash | 2 +- bin/ch-image.py.in | 7 ------- doc/ch-image.rst | 19 ++++++++----------- lib/build.py | 5 ----- 5 files changed, 13 insertions(+), 28 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f264b955d..5cc5e943c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -51,10 +51,10 @@ jobs: # only happen on CI. Comment out unless needed. WARNING: tmate.io has # access to unencrypted SSH traffic. # See: https://github.com/marketplace/actions/debugging-with-tmate - - name: set up tmate session - uses: mxschmitt/action-tmate@v3 - with: - detached: true + #- name: set up tmate session + # uses: mxschmitt/action-tmate@v3 + # with: + # detached: true - name: early setup & validation run: | diff --git a/bin/ch-completion.bash b/bin/ch-completion.bash index 92d2806d8..aa5c4c8d0 100644 --- a/bin/ch-completion.bash +++ b/bin/ch-completion.bash @@ -259,7 +259,7 @@ _ch_convert_complete () { _image_build_opts="-b --bind --build-arg -f --file --force --force-cmd -n --dry-run --parse-only -t --tag" -_image_modify_opts="-o --out" +_image_modify_opts="-c -S --shell" _image_common_opts="-a --arch --always-download --auth --break --cache --cache-large --dependencies -h --help diff --git a/bin/ch-image.py.in b/bin/ch-image.py.in index e5ef5d4b4..2d7546b86 100644 --- a/bin/ch-image.py.in +++ b/bin/ch-image.py.in @@ -283,16 +283,9 @@ def main(): sp = ap.add_parser("modify", "foo") add_opts(sp, build.modify, deps_check=True, stog_init=True) sp.add_argument("-c", metavar="CMD", action="append", default=[], nargs=1, help="foo") - # FIXME: Make “--out” optional - sp.add_argument("-o", "--out", metavar="out_image", help="foo", required=False) sp.add_argument("-S", "--shell", metavar="shell", help="foo") - #sp.add_argument("-u", "--unsafe", action="store_true", help="foo") sp.add_argument("image_ref", metavar="IMAGE_REF", help="image to modify") - # Using nargs="?" to make the positional argument optional (required is not a - # valid keyword for positionals). - #sp.add_argument("out_image", metavar="OUT_IMAGE", help="destination of modified image", dest="tag") sp.add_argument("out_image", metavar="OUT_IMAGE", help="destination of modified image") - #sp.add_argument("out_image", metavar="OUT_IMAGE", help="destination of modified image") # pull sp = ap.add_parser("pull", diff --git a/doc/ch-image.rst b/doc/ch-image.rst index 71c84f2db..d7c0a002a 100644 --- a/doc/ch-image.rst +++ b/doc/ch-image.rst @@ -1910,7 +1910,7 @@ Synopsis Description ----------- -This subcommand makes a copy of image :code:`TARGET`, named :code:`DEST`, and +This subcommand makes a copy of image :code:`TARGET` named :code:`DEST` and starts a shell on :code:`DEST` in order to edit the image interactively. It is similar to a :code:`RUN` instruction that starts an interactive shell. If there is already an image named :code:`DEST`, it will be overwritten. :code:`DEST` must @@ -1921,7 +1921,7 @@ be different than :code:`TARGET`. instruction in a Dockerfile. Can be repeated to run multiple commands sequentially. - :code:`-S SHELL` + :code:`-S`, :code:`--shell SHELL` Use shell :code:`SHELL` for interactive session, rather than the default :code:`/bin/sh`. @@ -1938,21 +1938,18 @@ Examples To edit the image :code:`foo`, adding :code:`/opt/lib` to the default shared library search path, producing image :code:`bar` as the result:: - $ ch-image modify bar foo - [...] - > emacs /etc/ld.so.conf - [... append line “/opt/lib” to the file ...] - > ldconfig + $ ch-image modify foo bar + copying image from cache ... + > echo foo >> /home/foo.txt > exit - committing ... - [...] + $ ch-run foo_modified -- cat /home/foo.txt + foo Equivalently, and almost certainly preferred:: $ cat Dockerfile FROM foo - RUN echo /opt/lib >> /etc/ld.so.conf - RUN ldconfig + RUN echo foo >> /home/foo.txt $ ch-image build -t bar -f Dockerfile . diff --git a/lib/build.py b/lib/build.py index af5e45bc7..55acfc5a4 100644 --- a/lib/build.py +++ b/lib/build.py @@ -297,12 +297,10 @@ def modify(cli_): # (if specified). cli.tag = str(out_image) - ch.ILLERI("CLI SHELL: %s" % cli.shell) if (cli.shell is not None): shell = cli.shell else: shell = "/bin/sh" - ch.ILLERI("SHELL: %s" % shell) # Second condition here is to ensure that “commands” does’t get overwritten # in the case where “ch-image modify” isn’t called from a terminal session # (e.g. in Github actions). I’m considering this a temporary fix, although I @@ -336,7 +334,6 @@ def modify(cli_): if (foo.returncode == 58): # FIXME: Write a better error message? ch.FATAL("Unable to run shell: %s" % shell) - ch.ILLERI("retcode: %s" % foo.returncode) ch.VERBOSE("using SID %s" % fake_sid) # FIXME: metadata history stuff? See misc.import_. if (out_image.metadata["history"] == []): @@ -379,9 +376,7 @@ def modify_tree_make(src_img, cmds): meta.line = -1 #df_children.append(im.Tree(lark.Token('RULE', 'from_'), [im.Tree(lark.Token('RULE', 'image_ref'),[lark.Token('IMAGE_REF', src_img.name)], meta)], meta)) df_children.append(im.Tree(lark.Token('RULE', 'from_'), [im.Tree(lark.Token('RULE', 'image_ref'),[lark.Token('IMAGE_REF', str(src_img))], meta)], meta)) - ch.ILLERI("ADDING COMMANDS TO TREE") for cmd in cmds: - ch.ILLERI("command: %s" % cmd) df_children.append(im.Tree(lark.Token('RULE', 'run'), [im.Tree(lark.Token('RULE', 'run_shell'),[lark.Token('LINE_CHUNK', cmd)], meta)],meta)) return im.Tree(lark.Token('RULE', 'start'), [im.Tree(lark.Token('RULE','dockerfile'), df_children)], meta) From 9d74d192550289e486ad2b519bc15c64b5130571 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Wed, 1 May 2024 16:32:49 +0000 Subject: [PATCH 28/65] put CI exit statuses in one place --- examples/multistage/test.bats | 2 +- test/build/50_ch-image.bats | 6 +-- test/common.bash | 5 ++ test/run/ch-run_escalated.bats | 6 +-- test/run/ch-run_join.bats | 36 +++++++-------- test/run/ch-run_misc.bats | 84 +++++++++++++++++----------------- 6 files changed, 72 insertions(+), 67 deletions(-) diff --git a/examples/multistage/test.bats b/examples/multistage/test.bats index 2a48f3e7a..14d4d3c1a 100644 --- a/examples/multistage/test.bats +++ b/examples/multistage/test.bats @@ -37,7 +37,7 @@ setup () { # Can’t run GCC. run ch-run "$ch_img" -- gcc --version echo "$output" - [[ $status -eq 58 ]] + [[ $status -eq $CH_ERR_CMD ]] [[ $output = *'gcc: No such file or directory'* ]] # No GCC or Make. diff --git a/test/build/50_ch-image.bats b/test/build/50_ch-image.bats index 347aa9f03..a94594c0b 100644 --- a/test/build/50_ch-image.bats +++ b/test/build/50_ch-image.bats @@ -864,17 +864,17 @@ EOF @test 'ch-run storage errors' { run ch-run -v -w alpine:3.17 -- /bin/true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *'error: --write invalid when running by name'* ]] run ch-run -v "$CH_IMAGE_STORAGE"/img/alpine+3.17 -- /bin/true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *"error: can't run directory images from storage (hint: run by name)"* ]] run ch-run -v -s /doesnotexist alpine:3.17 -- /bin/true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *'warning: storage directory not found: /doesnotexist'* ]] [[ $output = *"error: can't stat: alpine:3.17: No such file or directory"* ]] } diff --git a/test/common.bash b/test/common.bash index c9f7a1d30..68abb8622 100644 --- a/test/common.bash +++ b/test/common.bash @@ -329,6 +329,11 @@ chmod 700 "$btnew" export BATS_TMPDIR=$btnew [[ $(stat -c %a "$BATS_TMPDIR") = '700' ]] +# ch-run exit codes (see also: ch_misc.h) +export CH_ERR_RUN=57 +export CH_ERR_CMD=58 +export CH_ERR_SQUASH=59 # Currently not used, here just in case + ch_runfile=$(command -v ch-run) # Charliecloud version. diff --git a/test/run/ch-run_escalated.bats b/test/run/ch-run_escalated.bats index e90492ed8..548084e39 100644 --- a/test/run/ch-run_escalated.bats +++ b/test/run/ch-run_escalated.bats @@ -15,7 +15,7 @@ load ../common [[ -g $ch_run_tmp ]] run "$ch_run_tmp" --version echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *': please report this bug ('* ]] rm "$ch_run_tmp" } @@ -32,7 +32,7 @@ load ../common [[ -u $ch_run_tmp ]] run "$ch_run_tmp" --version echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *': please report this bug ('* ]] sudo rm "$ch_run_tmp" } @@ -71,7 +71,7 @@ load ../common fi run sudo -u root -g "$(id -gn)" "$ch_runfile" -v --version echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *'please report this bug ('* ]] } diff --git a/test/run/ch-run_join.bats b/test/run/ch-run_join.bats index 3a7555686..7521d3875 100644 --- a/test/run/ch-run_join.bats +++ b/test/run/ch-run_join.bats @@ -267,36 +267,36 @@ unset_vars () { # --join but no join count run ch-run --join "$ch_timg" -- true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output =~ 'join: no valid peer group size found' ]] ipc_clean_p # join count no digits run ch-run --join-ct=a "$ch_timg" -- true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output =~ 'join-ct: no digits found' ]] SLURM_CPUS_ON_NODE=a run ch-run --join "$ch_timg" -- true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output =~ 'SLURM_CPUS_ON_NODE: no digits found' ]] ipc_clean_p # join count empty string run ch-run --join-ct='' "$ch_timg" -- true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output =~ '--join-ct: no digits found' ]] SLURM_CPUS_ON_NODE=-1 run ch-run --join "$ch_timg" -- true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output =~ 'join: no valid peer group size found' ]] ipc_clean_p # --join-ct digits followed by extra goo (OK from environment variable) run ch-run --join-ct=1a "$ch_timg" -- true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output =~ '--join-ct: extra characters after digits' ]] ipc_clean_p @@ -306,48 +306,48 @@ unset_vars () { # join count above INT_MAX run ch-run --join-ct=2147483648 "$ch_timg" -- true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output =~ $range_re ]] SLURM_CPUS_ON_NODE=2147483648 \ run ch-run --join "$ch_timg" -- true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output =~ $range_re ]] ipc_clean_p # join count below INT_MIN run ch-run --join-ct=-2147483649 "$ch_timg" -- true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output =~ $range_re ]] SLURM_CPUS_ON_NODE=-2147483649 \ run ch-run --join "$ch_timg" -- true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output =~ $range_re ]] ipc_clean_p # join count above LONG_MAX run ch-run --join-ct=9223372036854775808 "$ch_timg" -- true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output =~ $range_re ]] SLURM_CPUS_ON_NODE=9223372036854775808 \ run ch-run --join "$ch_timg" -- true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output =~ $range_re ]] ipc_clean_p # join count below LONG_MIN run ch-run --join-ct=-9223372036854775809 "$ch_timg" -- true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output =~ $range_re ]] SLURM_CPUS_ON_NODE=-9223372036854775809 \ run ch-run --join "$ch_timg" -- true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output =~ $range_re ]] ipc_clean_p } @@ -361,11 +361,11 @@ unset_vars () { # join tag empty string run ch-run --join-tag='' "$ch_timg" -- true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output =~ 'join: peer group tag cannot be empty string' ]] SLURM_STEP_ID='' run ch-run --join "$ch_timg" -- true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output =~ 'join: peer group tag cannot be empty string' ]] ipc_clean_p } @@ -466,14 +466,14 @@ unset_vars () { # Can’t join namespaces of processes we don’t own. run ch-run -v --join-pid=1 "$ch_timg" -- true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *"join: can't open /proc/1/ns/user: Permission denied"* ]] # Can’t join namespaces of processes that don’t exist. pid=2147483647 run ch-run -v --join-pid="$pid" "$ch_timg" -- true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *"join: no PID ${pid}: /proc/${pid}/ns/user not found"* ]] } diff --git a/test/run/ch-run_misc.bats b/test/run/ch-run_misc.bats index 039d2e8cc..1d3960264 100644 --- a/test/run/ch-run_misc.bats +++ b/test/run/ch-run_misc.bats @@ -1,4 +1,4 @@ -load ../common + load ../common bind1_dir=$BATS_TMPDIR/bind1 bind2_dir=$BATS_TMPDIR/bind2 @@ -121,7 +121,7 @@ EOF run ch-run --home "$ch_timg" -- /bin/sh -c 'echo $HOME' export HOME="$home_tmp" echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] # shellcheck disable=SC2016 [[ $output = *'--home failed: $HOME not set'* ]] @@ -132,7 +132,7 @@ EOF run ch-run --home "$ch_timg" -- /bin/sh -c 'echo $HOME' export USER=$user_tmp echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] # shellcheck disable=SC2016 [[ $output = *'$USER not set'* ]] } @@ -210,7 +210,7 @@ EOF # Error if directory does not exist. run ch-run --cd /goops "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output =~ "can't cd to /goops: No such file or directory" ]] } @@ -324,7 +324,7 @@ EOF echo "STATUS" echo "$status" echo "STATUS" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *'--bind: no source provided'* ]] # source not provided @@ -333,106 +333,106 @@ EOF echo "STATUS" echo "$status" echo "STATUS" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *'--bind: no source provided'* ]] # destination not provided run ch-run -b "${bind1_dir}:" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *'--bind: no destination provided'* ]] # destination is / run ch-run -b "${bind1_dir}:/" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *"--bind: destination can't be /"* ]] # destination is relative run ch-run -b "${bind1_dir}:foo" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *"--bind: destination must be absolute"* ]] # destination climbs out of image, exists run ch-run -b "${bind1_dir}:/.." "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *"can't bind: "*"/${USER}.ch not subdirectory of "*"/${USER}.ch/mnt"* ]] # destination climbs out of image, does not exist run ch-run -b "${bind1_dir}:/../doesnotexist/a" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *"can't mkdir: "*"/${USER}.ch/doesnotexist not subdirectory of "*"/${USER}.ch/mnt"* ]] [[ ! -e ${ch_imgdir}/doesnotexist ]] # source does not exist run ch-run -b "/doesnotexist" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *"can't bind: source not found: /doesnotexist"* ]] # destination does not exist and image is not writeable run ch-run -b "${bind1_dir}:/doesnotexist" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *"can't mkdir: "*"/${USER}.ch/mnt/doesnotexist: Read-only file system"* ]] # neither source nor destination exist run ch-run -b /doesnotexist-out:/doesnotexist-in "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *"can't bind: source not found: /doesnotexist-out"* ]] # correct bind followed by source does not exist run ch-run -b "${bind1_dir}:/mnt/0" -b /doesnotexist "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *"can't bind: source not found: /doesnotexist"* ]] # correct bind followed by destination does not exist run ch-run -b "${bind1_dir}:/mnt/0" -b "${bind2_dir}:/doesnotexist" \ "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *"can't mkdir: "*"/${USER}.ch/mnt/doesnotexist: Read-only file system"* ]] # destination is broken symlink run ch-run -b "${bind1_dir}:/mnt/link-b0rken-abs" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *"can't mkdir: symlink not relative: "*"/${USER}.ch/mnt/mnt/link-b0rken-abs"* ]] # destination is absolute symlink outside image run ch-run -b "${bind1_dir}:/mnt/link-bad-abs" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *"can't bind: "*" not subdirectory of"* ]] # destination relative symlink outside image run ch-run -b "${bind1_dir}:/mnt/link-bad-rel" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *"can't bind: "*" not subdirectory of"* ]] # mkdir(2) under existing bind-mount, default, first level run ch-run -b "${bind1_dir}:/proc/doesnotexist" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *"can't mkdir: "*"/${USER}.ch/mnt/proc/doesnotexist under existing bind-mount "*"/${USER}.ch/mnt/proc "* ]] # mkdir(2) under existing bind-mount, user-supplied, first level run ch-run -b "${bind1_dir}:/mnt/0" \ -b "${bind2_dir}:/mnt/0/foo" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *"can't mkdir: "*"/${USER}.ch/mnt/mnt/0/foo under existing bind-mount "*"/${USER}.ch/mnt/mnt/0 "* ]] # mkdir(2) under existing bind-mount, default, 2nd level run ch-run -b "${bind1_dir}:/proc/sys/doesnotexist" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *"can't mkdir: "*"/${USER}.ch/mnt/proc/sys/doesnotexist under existing bind-mount "*"/${USER}.ch/mnt/proc "* ]] } @@ -629,13 +629,13 @@ EOF # file does not exist run ch-run --set-env=doesnotexist.txt "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *"can't open: doesnotexist.txt: No such file or directory"* ]] # /ch/environment missing run ch-run --set-env "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *"can't open: /ch/environment: No such file or directory"* ]] # Note: I’m not sure how to test an error during reading, i.e., getline(3) @@ -645,14 +645,14 @@ EOF echo 'FOO bar' > "$f_in" run ch-run --set-env="$f_in" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *"can't parse variable: no delimiter: ${f_in}:1"* ]] # invalid line: no name echo '=bar' > "$f_in" run ch-run --set-env="$f_in" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *"can't parse variable: empty name: ${f_in}:1"* ]] } @@ -671,7 +671,7 @@ EOF # missing environment variable run ch-run --set-env='$PATH:foo' "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *'$PATH:foo: No such file or directory'* ]] } @@ -718,7 +718,7 @@ EOF printf '\n# Empty string\n\n' run ch-run --unset-env= "$ch_timg" -- env echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *'--unset-env: GLOB must have non-zero length'* ]] } @@ -921,7 +921,7 @@ EOF # image is file but not sqfs run ch-run -vv ./fixtures/README -- /bin/true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output = *'magic expected: 6873 7173; actual: 596f 7520'* ]] [[ $output = *'unknown image type: '*'/fixtures/README'* ]] @@ -957,7 +957,7 @@ EOF # This should start up the container OK but fail to find the user command. run ch-run "$img" -- /bin/true echo "$output" - [[ $status -eq 58 ]] + [[ $status -eq $CH_ERR_CMD ]] [[ $output = *"can't execve(2): /bin/true: No such file or directory"* ]] # For each required file, we want a correct error if it’s missing. @@ -968,7 +968,7 @@ EOF run ch-run "$img" -- /bin/true touch "${img}/${f}" # restore before test fails for idempotency echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] r="can't bind: destination not found: .+/${f}" echo "expected: ${r}" [[ $output =~ $r ]] @@ -981,7 +981,7 @@ EOF run ch-run "$img" -- /bin/true touch "${img}/${f}" # restore before test fails for idempotency echo "$output" - [[ $status -eq 58 ]] + [[ $status -eq $CH_ERR_CMD ]] [[ $output = *"can't execve(2): /bin/true: No such file or directory"* ]] done @@ -994,7 +994,7 @@ EOF rmdir "${img}/${f}" # restore before test fails for idempotency touch "${img}/${f}" echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] r="can't bind .+ to /.+/${f}: Not a directory" echo "expected: ${r}" [[ $output =~ $r ]] @@ -1007,7 +1007,7 @@ EOF run ch-run "$img" -- /bin/true mkdir "${img}/${d}" # restore before test fails for idempotency echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] r="can't bind: destination not found: .+/${d}" echo "expected: ${r}" [[ $output =~ $r ]] @@ -1022,7 +1022,7 @@ EOF rm "${img}/${d}" # restore before test fails for idempotency mkdir "${img}/${d}" echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] r="can't bind .+ to /.+/${d}: Not a directory" echo "expected: ${r}" [[ $output =~ $r ]] @@ -1033,7 +1033,7 @@ EOF run ch-run --private-tmp "$img" -- /bin/true mkdir "${img}/tmp" # restore before test fails for idempotency echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] r="can't mount tmpfs at /.+/tmp: No such file or directory" echo "expected: ${r}" [[ $output =~ $r ]] @@ -1043,13 +1043,13 @@ EOF run ch-run "$img" -- /bin/true mkdir "${img}/home" # restore before test fails for idempotency echo "$output" - [[ $status -eq 58 ]] + [[ $status -eq $CH_ERR_CMD ]] [[ $output = *"can't execve(2): /bin/true: No such file or directory"* ]] # Everything should be restored and back to the original error. run ch-run "$img" -- /bin/true echo "$output" - [[ $status -eq 58 ]] + [[ $status -eq $CH_ERR_CMD ]] [[ $output = *"can't execve(2): /bin/true: No such file or directory"* ]] # At this point, there should be exactly two each of passwd and group @@ -1153,7 +1153,7 @@ EOF # subprocess failure at quiet level 2 run ch-run -qq "$ch_timg" -- doesnotexist echo "$output" - [[ $status -eq 58 ]] + [[ $status -eq $CH_ERR_CMD ]] [[ $output = *"error: can't execve(2): doesnotexist: No such file or directory"* ]] # quiet level 3 @@ -1166,12 +1166,12 @@ EOF # subprocess failure at quiet level 3 run ch-run -qqq "$ch_timg" -- doesnotexist echo "$output" - [[ $status -eq 58 ]] + [[ $status -eq $CH_ERR_CMD ]] [[ $output != *"error: can't execve(2): doesnotexist: No such file or directory"* ]] # failure at quiet level 3 run ch-run -qqq --test=log-fail - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output != *'info'* ]] [[ $output != *'warning: warning'* ]] [[ $output = *'error: the program failed inexplicably'* ]] @@ -1183,6 +1183,6 @@ EOF # bad tmpfs size run ch-run --write-fake=foo "$ch_timg" -- true echo "$output" - [[ $status -eq 57 ]] + [[ $status -eq $CH_ERR_RUN ]] [[ $output == *'cannot mount tmpfs for overlay: Invalid argument'* ]] } From 579ec81b6070cf83ff1f5ed3477b0aab08c04f9e Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Thu, 2 May 2024 18:14:10 +0000 Subject: [PATCH 29/65] ignore seemingly erroneous SC error --- test/common.bash | 6 +++--- test/run/ch-run_misc.bats | 6 +++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/test/common.bash b/test/common.bash index 68abb8622..81bc516ef 100644 --- a/test/common.bash +++ b/test/common.bash @@ -330,9 +330,9 @@ export BATS_TMPDIR=$btnew [[ $(stat -c %a "$BATS_TMPDIR") = '700' ]] # ch-run exit codes (see also: ch_misc.h) -export CH_ERR_RUN=57 -export CH_ERR_CMD=58 -export CH_ERR_SQUASH=59 # Currently not used, here just in case +CH_ERR_RUN=57 +CH_ERR_CMD=58 +CH_ERR_SQUASH=59 # Currently not used, here just in case ch_runfile=$(command -v ch-run) diff --git a/test/run/ch-run_misc.bats b/test/run/ch-run_misc.bats index 1d3960264..157695785 100644 --- a/test/run/ch-run_misc.bats +++ b/test/run/ch-run_misc.bats @@ -18,6 +18,7 @@ demand-overlayfs () { @test 'relative path to image' { # issue #6 scope full + # shellcheck disable=SC2154 cd "$(dirname "$ch_timg")" && ch-run "$(basename "$ch_timg")" -- /bin/true } @@ -86,7 +87,7 @@ EOF [[ $USER ]] # default: no change - # shellcheck disable=SC2016 + # shellcheck disable=SC2016,SC2154 run ch-run "${ch_imgdir}"/quick -- /bin/sh -c 'echo $HOME' echo "$output" [[ $status -eq 0 ]] @@ -142,6 +143,7 @@ EOF scope quick echo "$PATH" # if /bin is in $PATH, latter passes through unchanged + # shellcheck disable=SC2154 PATH2="$ch_bin:/bin:/usr/bin" echo "$PATH2" # shellcheck disable=SC2016 @@ -171,6 +173,7 @@ EOF scope standard old_path=$PATH unset PATH + # shellcheck disable=SC2154 run "$ch_runfile" "$ch_timg" -- \ /usr/bin/python3 -c 'import os; print(os.getenv("PATH") is None)' PATH=$old_path @@ -269,6 +272,7 @@ EOF } rm-img + # shellcheck disable=SC2154 ch-convert "$ch_tardir"/chtest.* "$img" ls -l "$img" mkdir "$img"/foo From b37232b2072dc1f83dee7e5048f08e69c891b5e7 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Thu, 2 May 2024 22:24:45 +0000 Subject: [PATCH 30/65] update exit codes and docs --- bin/ch-completion.bash | 8 +----- bin/ch_fuse.c | 2 +- bin/ch_misc.h | 9 +++---- doc/ch-image.rst | 56 +++++++++++++++++++++++++++++++++++++++--- doc/ch-run.rst | 8 +++--- test/common.bash | 8 +++--- 6 files changed, 68 insertions(+), 23 deletions(-) diff --git a/bin/ch-completion.bash b/bin/ch-completion.bash index aa5c4c8d0..6a56cb10b 100644 --- a/bin/ch-completion.bash +++ b/bin/ch-completion.bash @@ -392,13 +392,7 @@ _ch_image_complete () { fi ;; modify) - case "$prev" in - -o|--out) - # Can’t complete for this option - COMPREPLY=() - return 0 - ;; - esac + # FIXME: Implement extras="$extras $_image_modify_opts" ;; esac diff --git a/bin/ch_fuse.c b/bin/ch_fuse.c index 16641c823..0e1fdb19e 100644 --- a/bin/ch_fuse.c +++ b/bin/ch_fuse.c @@ -188,7 +188,7 @@ int sq_loop(void) // Clean up zombie child if exit signal was SIGCHLD. if (!sigchld_received) - exit_code = 59; + exit_code = ERR_SQUASH; else { Tf (wait(&child_status) >= 0, "can't wait for child"); if (WIFEXITED(child_status)) { diff --git a/bin/ch_misc.h b/bin/ch_misc.h index c86755236..c8b571fc2 100644 --- a/bin/ch_misc.h +++ b/bin/ch_misc.h @@ -24,10 +24,10 @@ don’t need to worry about running out of room. */ #define WARNINGS_SIZE (4*1024) -/* Exit codes */ -#define ERR_CHRUN 57 -#define ERR_CMD 58 -#define ERR_SQUASH 59 +/* Exit codes (see also: test/common.bash). */ +#define ERR_CHRUN 31 +#define ERR_CMD 49 +#define ERR_SQUASH 84 /* Test some value, and if it's not what we expect, exit with a fatal error. These are macros so we have access to the file and line number. @@ -67,7 +67,6 @@ #define T_(x) if (!(x)) msg_fatal(__FILE__, __LINE__, errno, NULL) #define Tf(x, ...) if (!(x)) msg_fatal(__FILE__, __LINE__, errno, __VA_ARGS__) #define Te(x, ...) if (!(x)) msg_fatal(__FILE__, __LINE__, 0, __VA_ARGS__) -#define Terror(x, ...) if (!(x)) msg_error(__FILE__, __LINE__, errno, __VA_ARGS__) #define Z_(x) if (x) msg_fatal(__FILE__, __LINE__, errno, NULL) #define Zf(x, ...) if (x) msg_fatal(__FILE__, __LINE__, errno, __VA_ARGS__) #define Ze(x, ...) if (x) msg_fatal(__FILE__, __LINE__, 0, __VA_ARGS__) diff --git a/doc/ch-image.rst b/doc/ch-image.rst index d7c0a002a..118bcdd61 100644 --- a/doc/ch-image.rst +++ b/doc/ch-image.rst @@ -1905,16 +1905,16 @@ Synopsis :: - $ ch-image [...] modify [...] TARGET DEST + $ ch-image [...] modify [...] SOURCE DEST Description ----------- -This subcommand makes a copy of image :code:`TARGET` named :code:`DEST` and +This subcommand makes a copy of image :code:`SOURCE` named :code:`DEST` and starts a shell on :code:`DEST` in order to edit the image interactively. It is similar to a :code:`RUN` instruction that starts an interactive shell. If there is already an image named :code:`DEST`, it will be overwritten. :code:`DEST` must -be different than :code:`TARGET`. +be different than :code:`SOURCE`. :code:`-c CMD` Run :code:`CMD` inside the container, as though specified by the :code:`RUN` @@ -1925,6 +1925,56 @@ be different than :code:`TARGET`. Use shell :code:`SHELL` for interactive session, rather than the default :code:`/bin/sh`. +Build cache implications +------------------------ + +Images produced by a :code:`modify` operation (:code:`DEST`) are adopted into +the build cache, with the parent being the specified source image +(:code:`SOURCE`). The nature of :code:`modify` makes it impossible to perform +the usual computation of state ID (SID) for the resulting cache entry without +intercepting the input of the interactive session, which is unnecessarily +complicated. Instead, we assign the adopted image a “fake” SID in the +form of a version 4 UUID. + +SIDs are used to retrieve image states from the cache. Since the :code:`modify` +operation is at odds with the notion of container provenance, image states +resulting from :code:`modify` will never be used by the cache, so using UUIDs in +place of SIDs does not undermine cache functionality. Adoption of modified +images is done soley for cache consistency rather than actual utility. + +Below is an example of cache adoption of a modified image, starting with a +non-empty cache:: + + $ ch-image build-cache --tree + * (bar) IMPORT bar + | * (foo) IMPORT foo + |/ + * (root) ROOT + + named images: 3 + state IDs: 3 + large files: 0 + commits: 3 + internal files: 7 K + disk used: 227 MiB + $ ch-image modify foo baz + copying image from cache ... + [...] + > exit + $ ch-image build-cache --tree + * (baz) MODIFY interactive + * (foo) IMPORT foo + | * (bar) IMPORT bar + |/ + * (root) ROOT + + named images: 4 + state IDs: 4 + large files: 0 + commits: 4 + internal files: 7 K + disk used: 227 MiB + .. warning:: This subcommand is rarely needed. Non-interactive build using a Dockerfile diff --git a/doc/ch-run.rst b/doc/ch-run.rst index 4f0256516..744a3d443 100644 --- a/doc/ch-run.rst +++ b/doc/ch-run.rst @@ -755,15 +755,17 @@ filesystem and the user command is killed by a signal, the exit status is 1 regardless of the signal value. Alternatively, :code:`ch-run` can exit with the following statuses: -57 +31 Error during containerization (:code:`ch-run` failure) -58 +49 Unable to start user command -59 +84 SquashFUSE loop exited on signal before user command was complete +128+N + Child process failed to exit normally, with exit code N .. include:: ./bugs.rst .. include:: ./see_also.rst diff --git a/test/common.bash b/test/common.bash index 81bc516ef..b5e48595d 100644 --- a/test/common.bash +++ b/test/common.bash @@ -329,10 +329,10 @@ chmod 700 "$btnew" export BATS_TMPDIR=$btnew [[ $(stat -c %a "$BATS_TMPDIR") = '700' ]] -# ch-run exit codes (see also: ch_misc.h) -CH_ERR_RUN=57 -CH_ERR_CMD=58 -CH_ERR_SQUASH=59 # Currently not used, here just in case +# ch-run exit codes. (see also: ch_misc.h) +CH_ERR_RUN=31 +CH_ERR_CMD=49 +CH_ERR_SQUASH=84 # Currently not used, here just in case ch_runfile=$(command -v ch-run) From 3742a288d18f91a7f2a80fa8284944a89556c4f3 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Mon, 6 May 2024 19:09:54 +0000 Subject: [PATCH 31/65] =?UTF-8?q?oops=20=F0=9F=A4=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/ch_misc.h | 2 +- lib/build.py | 4 +++- test/common.bash | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/bin/ch_misc.h b/bin/ch_misc.h index c8b571fc2..8eee99d1f 100644 --- a/bin/ch_misc.h +++ b/bin/ch_misc.h @@ -24,7 +24,7 @@ don’t need to worry about running out of room. */ #define WARNINGS_SIZE (4*1024) -/* Exit codes (see also: test/common.bash). */ +/* Exit codes (see also: test/common.bash, lib/build.py). */ #define ERR_CHRUN 31 #define ERR_CMD 49 #define ERR_SQUASH 84 diff --git a/lib/build.py b/lib/build.py index 55acfc5a4..728825ed4 100644 --- a/lib/build.py +++ b/lib/build.py @@ -331,7 +331,9 @@ def modify(cli_): bu.cache.branch_nocheckout(src_image.ref, out_image.ref) foo = subprocess.run([ch.CH_BIN + "/ch-run", "--unsafe", "-w", str(out_image.ref), "--", shell]) - if (foo.returncode == 58): + # FIXME: This causes issues when you change the value in ch_misc.h and + # forget to change it here... + if (foo.returncode == 49): # FIXME: Write a better error message? ch.FATAL("Unable to run shell: %s" % shell) ch.VERBOSE("using SID %s" % fake_sid) diff --git a/test/common.bash b/test/common.bash index b5e48595d..4b17a4281 100644 --- a/test/common.bash +++ b/test/common.bash @@ -329,7 +329,7 @@ chmod 700 "$btnew" export BATS_TMPDIR=$btnew [[ $(stat -c %a "$BATS_TMPDIR") = '700' ]] -# ch-run exit codes. (see also: ch_misc.h) +# ch-run exit codes. (see also: ch_misc.h, lib/build.py) CH_ERR_RUN=31 CH_ERR_CMD=49 CH_ERR_SQUASH=84 # Currently not used, here just in case From 6e0ff695dc87e0918275db7db99a6c4b687409ae Mon Sep 17 00:00:00 2001 From: Reid Priedhorsky Date: Fri, 10 May 2024 11:31:04 -0600 Subject: [PATCH 32/65] tidy docs [skip ci] --- doc/ch-image.rst | 299 ++++++++++++++++++++++++++++++----------------- doc/ch-run.rst | 33 +++--- 2 files changed, 207 insertions(+), 125 deletions(-) diff --git a/doc/ch-image.rst b/doc/ch-image.rst index 118bcdd61..04731d3f2 100644 --- a/doc/ch-image.rst +++ b/doc/ch-image.rst @@ -1895,114 +1895,6 @@ functionally identical when re-imported. will still hit, even if the new image is different. -:code:`modify` -============== - -Interactively edit the specified image. - -Synopsis --------- - -:: - - $ ch-image [...] modify [...] SOURCE DEST - -Description ------------ - -This subcommand makes a copy of image :code:`SOURCE` named :code:`DEST` and -starts a shell on :code:`DEST` in order to edit the image interactively. It is -similar to a :code:`RUN` instruction that starts an interactive shell. If there -is already an image named :code:`DEST`, it will be overwritten. :code:`DEST` must -be different than :code:`SOURCE`. - - :code:`-c CMD` - Run :code:`CMD` inside the container, as though specified by the :code:`RUN` - instruction in a Dockerfile. Can be repeated to run multiple commands - sequentially. - - :code:`-S`, :code:`--shell SHELL` - Use shell :code:`SHELL` for interactive session, rather than the default - :code:`/bin/sh`. - -Build cache implications ------------------------- - -Images produced by a :code:`modify` operation (:code:`DEST`) are adopted into -the build cache, with the parent being the specified source image -(:code:`SOURCE`). The nature of :code:`modify` makes it impossible to perform -the usual computation of state ID (SID) for the resulting cache entry without -intercepting the input of the interactive session, which is unnecessarily -complicated. Instead, we assign the adopted image a “fake” SID in the -form of a version 4 UUID. - -SIDs are used to retrieve image states from the cache. Since the :code:`modify` -operation is at odds with the notion of container provenance, image states -resulting from :code:`modify` will never be used by the cache, so using UUIDs in -place of SIDs does not undermine cache functionality. Adoption of modified -images is done soley for cache consistency rather than actual utility. - -Below is an example of cache adoption of a modified image, starting with a -non-empty cache:: - - $ ch-image build-cache --tree - * (bar) IMPORT bar - | * (foo) IMPORT foo - |/ - * (root) ROOT - - named images: 3 - state IDs: 3 - large files: 0 - commits: 3 - internal files: 7 K - disk used: 227 MiB - $ ch-image modify foo baz - copying image from cache ... - [...] - > exit - $ ch-image build-cache --tree - * (baz) MODIFY interactive - * (foo) IMPORT foo - | * (bar) IMPORT bar - |/ - * (root) ROOT - - named images: 4 - state IDs: 4 - large files: 0 - commits: 4 - internal files: 7 K - disk used: 227 MiB - -.. warning:: - - This subcommand is rarely needed. Non-interactive build using a Dockerfile - is almost always better, because it preserves the sequence of operations - that created an image. Only use this subcommand if you really know what you - are doing. - -Examples --------- - -To edit the image :code:`foo`, adding :code:`/opt/lib` to the default shared -library search path, producing image :code:`bar` as the result:: - - $ ch-image modify foo bar - copying image from cache ... - > echo foo >> /home/foo.txt - > exit - $ ch-run foo_modified -- cat /home/foo.txt - foo - -Equivalently, and almost certainly preferred:: - - $ cat Dockerfile - FROM foo - RUN echo foo >> /home/foo.txt - $ ch-image build -t bar -f Dockerfile . - - :code:`pull` ============ @@ -2214,6 +2106,195 @@ in the remote registry, so we don’t upload it again.) Delete all images and cache from ch-image builder storage. +:code:`shell` +============= + +Modify an image with shell commands, possibly interactively. + +Synopsis +-------- + +:: + + $ ch-image [...] shell [...] SOURCE DEST [SCRIPT] + +Description +----------- + +This subcommand modifies :code:`SOURCE` using shell commands to create +:code:`DEST`. These commands can be provided either interactively +(discouraged) or non-interactively. In the non-interactive case, the commands +are repackaged internally into a Dockerfile. + +Options: + + :code:`-c CMD` + Run :code:`CMD` as though specified by a :code:`RUN` instruction. Can be + repeated to run multiple commands sequentially. + + :code:`-i` + Execute the shell in interactive mode (by specifying :code:`-i` to it) + even if standard input is not a TTY. + + :code:`-S`, :code:`--shell SHELL` + Use :code:`SHELL` instead of the default :code:`/bin/sh`. + +:code:`ch-image shell` operates in one of the following three modes. If the +mode desired is ambiguous, that is an error. + +Non-interactive mode, commands specified with :code:`-c` +-------------------------------------------------------- + +The following are equivalent:: + + $ ch-image modify -S /bin/ash -c 'echo hello' -c 'echo world' foo bar + +and:: + + $ ch-image build -t bar <<'EOF' + FROM foo + SHELL /bin/ash + RUN echo hello + RUN echo world + EOF + +That is, :code:`ch-image` simply builds a Dockerfile internally that uses +:code:`foo` as a base image, starts with an appropriate :code:`SHELL` if +:code:`-S` was given, converts each :code:`-c` to a :code:`RUN` command, and +executes this Dockerfile to produce image :code:`bar`. That is, if any command +fails, the build fails and no further commands are attempted. + +This mode provides a detailed image provenance just like a Dockerfile. + +Non-interactive mode using a shell script +----------------------------------------- + +The following are equivalent:: + + $ ch-image shell foo bar /baz/qux.sh + +and:: + + $ ch-image build -t bar -f - / <<'EOF' + FROM foo + COPY /baz/qux.sh /ch/script.sh + RUN /bin/sh /ch/script.sh + +That is, :code:`ch-image` uses :code:`COPY` to put the script inside the +image, then runs it. + +If :code:`SCRIPT` is not provided and standard input is not a TTY, a script is +read from there instead. In this case, standard input is copied in full to a +file in a temporary directory, which is used as the context. The file’s +modification time is set to 1993-10-21T10:00:00Z and its name to a hash of the +content, so the cache hits if the content is the same and misses if not. That +is, the following are equivalent:: + + $ ch-image shell foo bar <<'EOF' + echo hello world + EOF + +and:: + + $ ctx=$(mktemp -d) + $ cat > $ctx/foo <<'EOF' + echo hello world + EOF + $ hash=$(md5sum $ctx/foo) + $ mv $ctx/foo $ctx/$hash + $ touch -d 1993-10-21T10:00:00Z $ctx/$hash + $ ch-image build -t bar -f - $ctx < exit + $ ch-image build-cache --tree + * (baz) MODIFY interactive + * (foo) IMPORT foo + | * (bar) IMPORT bar + |/ + * (root) ROOT + + named images: 4 + state IDs: 4 + large files: 0 + commits: 4 + internal files: 7 K + disk used: 227 MiB + + :code:`undelete` ================ @@ -2243,4 +2324,4 @@ Environment variables .. LocalWords: dlcache graphviz packfile packfiles bigFileThreshold fd Tpdf .. LocalWords: pstats gprof chofile cffd cacdb ARGs NSYNC dst imgroot popt .. LocalWords: globbed ni AHSXpr drwxrwx ctx sym nom newB newC newD dstC -.. LocalWords: dstB dstF dstG upover drwx kexec pdb +.. LocalWords: dstB dstF dstG upover drwx kexec pdb mktemp diff --git a/doc/ch-run.rst b/doc/ch-run.rst index 744a3d443..1515ae5ac 100644 --- a/doc/ch-run.rst +++ b/doc/ch-run.rst @@ -749,26 +749,27 @@ would terminate the string. Exit status =========== -If the user command is started successfully, the exit status is that of the user -command, with one exception: if the image is an internally mounted SquashFS -filesystem and the user command is killed by a signal, the exit status is 1 -regardless of the signal value. Alternatively, :code:`ch-run` can exit with the -following statuses: +If the user command is started successfully and exits normally, +:code:`ch-run`’s exit status is that of the user command. Otherwise, the exit +status is one of: -31 - Error during containerization (:code:`ch-run` failure) - -49 - Unable to start user command - -84 - SquashFUSE loop exited on signal before user command was complete - -128+N - Child process failed to exit normally, with exit code N +.. list-table:: + :header-rows: 0 + + * - 31 + - Miscellaneous :code:`ch-run` failure other than the below + * - 49 + - Unable to start user command (i.e., :code:`execvp(2)` failed) + * - 84 + - SquashFUSE loop exited on signal before user command was complete + * - 87 + - Feature queried by :code:`--feature` is not available + * - 128 + *N* + - User command killed by signal *N* .. include:: ./bugs.rst .. include:: ./see_also.rst + .. LocalWords: mtune NEWROOT hugetlbfs UsrMerge fusermount mybox IMG HOSTPATH .. LocalWords: noprofile norc SHLVL PWD kernelnewbies extglob From 26ce9e67d48311526a5d2558338748f1c190fd03 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Tue, 21 May 2024 21:50:09 +0000 Subject: [PATCH 33/65] work on suggestions --- bin/ch-image.py.in | 2 ++ doc/ch-image.rst | 14 ++++----- lib/build.py | 60 +++++++++++++++++++++++++++++-------- test/build/50_ch-image.bats | 4 ++- 4 files changed, 59 insertions(+), 21 deletions(-) diff --git a/bin/ch-image.py.in b/bin/ch-image.py.in index 2d7546b86..31227d783 100644 --- a/bin/ch-image.py.in +++ b/bin/ch-image.py.in @@ -286,6 +286,8 @@ def main(): sp.add_argument("-S", "--shell", metavar="shell", help="foo") sp.add_argument("image_ref", metavar="IMAGE_REF", help="image to modify") sp.add_argument("out_image", metavar="OUT_IMAGE", help="destination of modified image") + # Optional positional argument? https://stackoverflow.com/a/4480202 + sp.add_argument("script", metavar="SCRIPT", help="foo", nargs='?') # pull sp = ap.add_parser("pull", diff --git a/doc/ch-image.rst b/doc/ch-image.rst index 04731d3f2..8087765dc 100644 --- a/doc/ch-image.rst +++ b/doc/ch-image.rst @@ -2139,7 +2139,7 @@ Options: :code:`-S`, :code:`--shell SHELL` Use :code:`SHELL` instead of the default :code:`/bin/sh`. -:code:`ch-image shell` operates in one of the following three modes. If the +:code:`ch-image modify` operates in one of the following three modes. If the mode desired is ambiguous, that is an error. Non-interactive mode, commands specified with :code:`-c` @@ -2147,7 +2147,7 @@ Non-interactive mode, commands specified with :code:`-c` The following are equivalent:: - $ ch-image modify -S /bin/ash -c 'echo hello' -c 'echo world' foo bar + $ ch-image modify -S /bin/bash -c 'echo hello' -c 'echo world' foo bar and:: @@ -2171,7 +2171,7 @@ Non-interactive mode using a shell script The following are equivalent:: - $ ch-image shell foo bar /baz/qux.sh + $ ch-image modify foo bar /baz/qux.sh and:: @@ -2190,7 +2190,7 @@ modification time is set to 1993-10-21T10:00:00Z and its name to a hash of the content, so the cache hits if the content is the same and misses if not. That is, the following are equivalent:: - $ ch-image shell foo bar <<'EOF' + $ ch-image modify foo bar <<'EOF' echo hello world EOF @@ -2239,10 +2239,10 @@ Interactive mode shell script. Only use it if you really know what you are doing. If :code:`SCRIPT` is not provided and standard input *is* a TTY, -:code:`ch-image shell` opens an interactive shell. That is, the following are +:code:`ch-image modify` opens an interactive shell. That is, the following are roughly equivalent (assuming a terminal):: - $ ch-image shell foo bar + $ ch-image modify foo bar and:: @@ -2256,7 +2256,7 @@ empty root image. This mode largely defeats the cache. While images descending from :code:`DEST` will use the latest version (as :code:`FROM` normally behaves), no other -operations can re-use the results with a cache hit, and :code:`ch-image shell` +operations can re-use the results with a cache hit, and :code:`ch-image modify` cannot have a cache hit itself. (This is implemented with a random state ID.) We reasoned that making an interactive session cache-aware would be too difficult both conceptually and to implement. The mode is much like diff --git a/lib/build.py b/lib/build.py index 728825ed4..51b838f7a 100644 --- a/lib/build.py +++ b/lib/build.py @@ -279,6 +279,7 @@ def modify(cli_): cli.force = ch.Force_Mode.SECCOMP cli.force_cmd = force.FORCE_CMD_DEFAULT cli.bind = [] + cli.context = os.path.abspath(os.sep) commands = [] # “Flatten” commands array @@ -290,6 +291,9 @@ def modify(cli_): ch.FATAL("not in storage: %s" % src_image.ref) if (cli.out_image == cli.image_ref): ch.FATAL("output must be different from source image (%s)" % cli.image_ref) + if (cli.script is not None): + if (not ch.Path(cli.script).exists): + ch.FATAL("%s: no such file" % cli.script) # This kludge is necessary because cli is a global variable, with cli.tag # assumed present elsewhere in the file. cli.tag represents the image being @@ -301,18 +305,12 @@ def modify(cli_): shell = cli.shell else: shell = "/bin/sh" - # Second condition here is to ensure that “commands” does’t get overwritten - # in the case where “ch-image modify” isn’t called from a terminal session - # (e.g. in Github actions). I’m considering this a temporary fix, although I - # think a case could also be made for “-c” to have precedence over pipeline - # input. - if ((not sys.stdin.isatty()) and (commands == [])): - # Treat stdin as opaque blob and run that - commands = [sys.stdin.read()] - # Again, the fact that we have to check against both “[]” and “['']” is to - # get CI to work... - if (commands not in [[],['']]): - tree = modify_tree_make(src_image.ref, commands) + if ((commands not in [[],['']]) or (cli.script is not None)): + # FIXME: Kludge!!! + if (cli.script is not None): + tree = modify_tree_make_script(src_image.ref, cli.script) + else: + tree = modify_tree_make(src_image.ref, commands) # Count the number of stages (i.e., FROM instructions) global image_ct @@ -351,7 +349,7 @@ def modify_tree_make(src_img, cmds): more commands inside a container, the only Dockerfile instructions we need to consider are “FROM” and “RUN”. E.g. for the command line - $ ch-image modify -o foo2 -c 'echo foo' -c 'echo bar' -- foo + $ ch-image modify -c 'echo foo' -c 'echo bar' -- foo foo2 this function produces the following parse tree @@ -382,6 +380,42 @@ def modify_tree_make(src_img, cmds): df_children.append(im.Tree(lark.Token('RULE', 'run'), [im.Tree(lark.Token('RULE', 'run_shell'),[lark.Token('LINE_CHUNK', cmd)], meta)],meta)) return im.Tree(lark.Token('RULE', 'start'), [im.Tree(lark.Token('RULE','dockerfile'), df_children)], meta) +# FIXME: Probably should merge into “modify_tree_make” +# (This is a tmp function to implement functionality) +def modify_tree_make_script(src_img, path): + """Temporary(?) analog of “modify_tree_make” for the non-interactive version + of “modify” using a script. For the command line: + + $ ch-image modify foo foo2 /path/to/script + + this function produces the following parse tree + + start + dockerfile + from_ + image_ref + IMAGE_REF foo + copy + copy_shell + WORD /path/to/script WORD /ch/script.sh + run + run_shell + LINE_CHUNK /ch/script.sh + """ + # Children of dockerfile tree + df_children = [] + # Metadata attribute. We use this attribute in the “_pretty” method for our + # “Tree” class. Constructing a tree without specifying a “Meta” instance that + # has been given a “line” value will result in the attribute not being present, + # which causes an error when we try to access that attribute. Here we give the + # attribute a debug value of -1 to avoid said errors. + meta = lark.tree.Meta() + meta.line = -1 + df_children.append(im.Tree(lark.Token('RULE', 'from_'), [im.Tree(lark.Token('RULE', 'image_ref'),[lark.Token('IMAGE_REF', str(src_img))], meta)], meta)) + df_children.append(im.Tree(lark.Token('RULE', 'copy'), [im.Tree(lark.Token('RULE', 'copy_shell'),[lark.Token('WORD', path),lark.Token('WORD', '/ch/script.sh')], meta)],meta)) + df_children.append(im.Tree(lark.Token('RULE', 'run'), [im.Tree(lark.Token('RULE', 'run_shell'),[lark.Token('LINE_CHUNK', '/ch/script.sh')], meta)],meta)) + return im.Tree(lark.Token('RULE', 'start'), [im.Tree(lark.Token('RULE','dockerfile'), df_children)], meta) + # Traverse Lark parse tree and do what it says. # # We don’t actually care whether the tree is traversed breadth-first or diff --git a/test/build/50_ch-image.bats b/test/build/50_ch-image.bats index a94594c0b..334b1b333 100644 --- a/test/build/50_ch-image.bats +++ b/test/build/50_ch-image.bats @@ -994,7 +994,9 @@ EOF [[ $status -eq 0 ]] [[ $output = *'foo'* ]] - printf "touch /home/bar" | ch-image modify alpine:3.17 tmpimg + echo "touch /home/bar" >> "${BATS_TMPDIR}/modify-script.sh" + chmod 755 "${BATS_TMPDIR}/modify-script.sh" + ch-image modify alpine:3.17 tmpimg "${BATS_TMPDIR}/modify-script.sh" run ch-run tmpimg -- ls /home echo "$output" [[ $status -eq 0 ]] From dbbf74da5f22e3c406d14562bda7a6782b360336 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Tue, 4 Jun 2024 18:19:53 +0000 Subject: [PATCH 34/65] more work on non-interactive script case --- doc/ch-image.rst | 2 +- lib/build.py | 20 ++++++++++++++++++++ test/build/50_ch-image.bats | 14 ++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/doc/ch-image.rst b/doc/ch-image.rst index 8087765dc..77278dc67 100644 --- a/doc/ch-image.rst +++ b/doc/ch-image.rst @@ -2147,7 +2147,7 @@ Non-interactive mode, commands specified with :code:`-c` The following are equivalent:: - $ ch-image modify -S /bin/bash -c 'echo hello' -c 'echo world' foo bar + $ ch-image modify -S /bin/ash -c 'echo hello' -c 'echo world' foo bar and:: diff --git a/lib/build.py b/lib/build.py index 51b838f7a..fd98fecea 100644 --- a/lib/build.py +++ b/lib/build.py @@ -301,6 +301,19 @@ def modify(cli_): # (if specified). cli.tag = str(out_image) + if ((not sys.stdin.isatty()) and (commands == [])): + # https://stackoverflow.com/a/6482200 + stdin = sys.stdin.read() + # We use “decode("utf-8")” here because stdout seems default to a bytes + # object, which is not a valid type for an argument for “Path”. + tmpfile = ch.Path(subprocess.run(["mktemp", "-d"],capture_output=True).stdout.decode("utf-8")) + with open(tmpfile, "w") as outfile: + outfile.write(stdin) + # By default, the file is seemingly created with its execute bit + # unassigned. This is problematic for the RUN instruction. + os.chmod(tmpfile, 0o755) + cli.script = str(tmpfile) + if (cli.shell is not None): shell = cli.shell else: @@ -376,6 +389,10 @@ def modify_tree_make(src_img, cmds): meta.line = -1 #df_children.append(im.Tree(lark.Token('RULE', 'from_'), [im.Tree(lark.Token('RULE', 'image_ref'),[lark.Token('IMAGE_REF', src_img.name)], meta)], meta)) df_children.append(im.Tree(lark.Token('RULE', 'from_'), [im.Tree(lark.Token('RULE', 'image_ref'),[lark.Token('IMAGE_REF', str(src_img))], meta)], meta)) + if (cli.shell is not None): + #df_children.append(im.Tree(lark.Token('RULE', 'shell'), [lark.Token('STRING_QUO', "/bin/sh"),lark.Token('STRING_QUO', "-c")], meta)) + #ch.ILLERI("HERE") + df_children.append(im.Tree(lark.Token('RULE', 'shell'), [lark.Token('STRING_QUOTED', '"%s"' % cli.shell),lark.Token('STRING_QUOTED', '"-c"')],meta)) for cmd in cmds: df_children.append(im.Tree(lark.Token('RULE', 'run'), [im.Tree(lark.Token('RULE', 'run_shell'),[lark.Token('LINE_CHUNK', cmd)], meta)],meta)) return im.Tree(lark.Token('RULE', 'start'), [im.Tree(lark.Token('RULE','dockerfile'), df_children)], meta) @@ -412,6 +429,9 @@ def modify_tree_make_script(src_img, path): meta = lark.tree.Meta() meta.line = -1 df_children.append(im.Tree(lark.Token('RULE', 'from_'), [im.Tree(lark.Token('RULE', 'image_ref'),[lark.Token('IMAGE_REF', str(src_img))], meta)], meta)) + if (cli.shell is not None): + #df_children.append(im.Tree(lark.Token('RULE', 'from_'), [im.Tree(lark.Token('RULE', 'shell'),[lark.Token('STRING_QUO', "%s" % cli.shell),lark.Token('STRING_QUO', "-c")], meta)], meta)) + df_children.append(im.Tree(lark.Token('RULE', 'shell'), [lark.Token('STRING_QUOTED', '"%s"' % cli.shell),lark.Token('STRING_QUOTED', '"-c"')],meta)) df_children.append(im.Tree(lark.Token('RULE', 'copy'), [im.Tree(lark.Token('RULE', 'copy_shell'),[lark.Token('WORD', path),lark.Token('WORD', '/ch/script.sh')], meta)],meta)) df_children.append(im.Tree(lark.Token('RULE', 'run'), [im.Tree(lark.Token('RULE', 'run_shell'),[lark.Token('LINE_CHUNK', '/ch/script.sh')], meta)],meta)) return im.Tree(lark.Token('RULE', 'start'), [im.Tree(lark.Token('RULE','dockerfile'), df_children)], meta) diff --git a/test/build/50_ch-image.bats b/test/build/50_ch-image.bats index 334b1b333..389802ab9 100644 --- a/test/build/50_ch-image.bats +++ b/test/build/50_ch-image.bats @@ -982,18 +982,21 @@ EOF @test "ch-image modify" { + # -c success, echo run ch-image modify -c "echo foo" -c "echo bar" -- alpine:3.17 tmpimg echo "$output" [[ $status -eq 0 ]] [[ $output = *'foo'* ]] [[ $output = *'bar'* ]] + # -c success, create file ch-image modify -c "touch /home/foo" -- alpine:3.17 tmpimg run ch-run tmpimg -- ls /home echo "$output" [[ $status -eq 0 ]] [[ $output = *'foo'* ]] + # non-interactive, script echo "touch /home/bar" >> "${BATS_TMPDIR}/modify-script.sh" chmod 755 "${BATS_TMPDIR}/modify-script.sh" ch-image modify alpine:3.17 tmpimg "${BATS_TMPDIR}/modify-script.sh" @@ -1002,11 +1005,22 @@ EOF [[ $status -eq 0 ]] [[ $output = *'bar'* ]] + # non-interactive, here doc + ch-image modify alpine:3.17 tmpimg <<'EOF' +touch /home/foobar +EOF + run ch-run tmpimg -- ls /home + echo "$output" + [[ $status -eq 0 ]] + [[ $output = *'foobar'* ]] + + # -c fail run ch-image modify -c 'echo foo' -- alpine:3.17 alpine:3.17 echo "$output" [[ $status -eq 1 ]] [[ $output = *'output must be different from source image'* ]] + # non-existant shell run ch-image modify -S "doesnotexist" -- alpine:3.17 tmpimg echo "$output" [[ $status -eq 1 ]] From 8cb79752ecace325a198d7a71e79fa4f6718827a Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Tue, 4 Jun 2024 23:59:57 +0000 Subject: [PATCH 35/65] try to debug CI --- bin/ch-image.py.in | 1 + lib/build.py | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/bin/ch-image.py.in b/bin/ch-image.py.in index 31227d783..d043c17e2 100644 --- a/bin/ch-image.py.in +++ b/bin/ch-image.py.in @@ -284,6 +284,7 @@ def main(): add_opts(sp, build.modify, deps_check=True, stog_init=True) sp.add_argument("-c", metavar="CMD", action="append", default=[], nargs=1, help="foo") sp.add_argument("-S", "--shell", metavar="shell", help="foo") + sp.add_argument("--ci-debug", action="store_true", help="CI debugging, remove this before merge!") sp.add_argument("image_ref", metavar="IMAGE_REF", help="image to modify") sp.add_argument("out_image", metavar="OUT_IMAGE", help="destination of modified image") # Optional positional argument? https://stackoverflow.com/a/4480202 diff --git a/lib/build.py b/lib/build.py index fd98fecea..d488ef9a7 100644 --- a/lib/build.py +++ b/lib/build.py @@ -301,9 +301,21 @@ def modify(cli_): # (if specified). cli.tag = str(out_image) + ch.ILLERI(cli.script) + + if (cli.ci_debug): + stdin = sys.stdin.read() + ch.INFO("STDIN!!!") + ch.INFO(stdin) + if ((not sys.stdin.isatty()) and (commands == [])): # https://stackoverflow.com/a/6482200 - stdin = sys.stdin.read() + + # FIXME: remove this try except kludge + try: + ch.DEBUG(stdin) + except NameError: + stdin = sys.stdin.read() # We use “decode("utf-8")” here because stdout seems default to a bytes # object, which is not a valid type for an argument for “Path”. tmpfile = ch.Path(subprocess.run(["mktemp", "-d"],capture_output=True).stdout.decode("utf-8")) From b8590895c894f911345299d65b6337ec65486c74 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Wed, 5 Jun 2024 15:31:16 +0000 Subject: [PATCH 36/65] hopefully this works --- lib/build.py | 14 +++++--------- test/build/50_ch-image.bats | 2 +- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/lib/build.py b/lib/build.py index d488ef9a7..c7d16c528 100644 --- a/lib/build.py +++ b/lib/build.py @@ -303,19 +303,15 @@ def modify(cli_): ch.ILLERI(cli.script) - if (cli.ci_debug): - stdin = sys.stdin.read() - ch.INFO("STDIN!!!") - ch.INFO(stdin) if ((not sys.stdin.isatty()) and (commands == [])): # https://stackoverflow.com/a/6482200 - # FIXME: remove this try except kludge - try: - ch.DEBUG(stdin) - except NameError: - stdin = sys.stdin.read() + stdin = sys.stdin.read() + + if (cli.ci_debug): + ch.INFO("STDIN!!!") + ch.INFO(stdin) # We use “decode("utf-8")” here because stdout seems default to a bytes # object, which is not a valid type for an argument for “Path”. tmpfile = ch.Path(subprocess.run(["mktemp", "-d"],capture_output=True).stdout.decode("utf-8")) diff --git a/test/build/50_ch-image.bats b/test/build/50_ch-image.bats index 389802ab9..1355f4a5a 100644 --- a/test/build/50_ch-image.bats +++ b/test/build/50_ch-image.bats @@ -999,7 +999,7 @@ EOF # non-interactive, script echo "touch /home/bar" >> "${BATS_TMPDIR}/modify-script.sh" chmod 755 "${BATS_TMPDIR}/modify-script.sh" - ch-image modify alpine:3.17 tmpimg "${BATS_TMPDIR}/modify-script.sh" + ch-image modify --ci-debug alpine:3.17 tmpimg "${BATS_TMPDIR}/modify-script.sh" run ch-run tmpimg -- ls /home echo "$output" [[ $status -eq 0 ]] From 54c2ee8cb2ec1a07ca663d2f5d8c21563ae0e3e7 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Wed, 5 Jun 2024 16:28:29 +0000 Subject: [PATCH 37/65] back to how it used to work? --- bin/ch-image.py.in | 1 + lib/build.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/ch-image.py.in b/bin/ch-image.py.in index d043c17e2..e3a12fb19 100644 --- a/bin/ch-image.py.in +++ b/bin/ch-image.py.in @@ -284,6 +284,7 @@ def main(): add_opts(sp, build.modify, deps_check=True, stog_init=True) sp.add_argument("-c", metavar="CMD", action="append", default=[], nargs=1, help="foo") sp.add_argument("-S", "--shell", metavar="shell", help="foo") + # FIXME: remove sp.add_argument("--ci-debug", action="store_true", help="CI debugging, remove this before merge!") sp.add_argument("image_ref", metavar="IMAGE_REF", help="image to modify") sp.add_argument("out_image", metavar="OUT_IMAGE", help="destination of modified image") diff --git a/lib/build.py b/lib/build.py index c7d16c528..1992ee822 100644 --- a/lib/build.py +++ b/lib/build.py @@ -304,11 +304,13 @@ def modify(cli_): ch.ILLERI(cli.script) - if ((not sys.stdin.isatty()) and (commands == [])): + if ((not sys.stdin.isatty()) and (commands == []) and (not cli.ci_debug)): # https://stackoverflow.com/a/6482200 stdin = sys.stdin.read() + + # FIXME: remove if (cli.ci_debug): ch.INFO("STDIN!!!") ch.INFO(stdin) From bb0283e6f9dea46366f862046818c5648c0b1e62 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Wed, 5 Jun 2024 16:43:57 +0000 Subject: [PATCH 38/65] more interactive shell troubles --- bin/ch-image.py.in | 2 +- test/build/50_ch-image.bats | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/ch-image.py.in b/bin/ch-image.py.in index e3a12fb19..a34f1e457 100644 --- a/bin/ch-image.py.in +++ b/bin/ch-image.py.in @@ -284,7 +284,7 @@ def main(): add_opts(sp, build.modify, deps_check=True, stog_init=True) sp.add_argument("-c", metavar="CMD", action="append", default=[], nargs=1, help="foo") sp.add_argument("-S", "--shell", metavar="shell", help="foo") - # FIXME: remove + # FIXME: rename sp.add_argument("--ci-debug", action="store_true", help="CI debugging, remove this before merge!") sp.add_argument("image_ref", metavar="IMAGE_REF", help="image to modify") sp.add_argument("out_image", metavar="OUT_IMAGE", help="destination of modified image") diff --git a/test/build/50_ch-image.bats b/test/build/50_ch-image.bats index 1355f4a5a..bec216461 100644 --- a/test/build/50_ch-image.bats +++ b/test/build/50_ch-image.bats @@ -1021,7 +1021,7 @@ EOF [[ $output = *'output must be different from source image'* ]] # non-existant shell - run ch-image modify -S "doesnotexist" -- alpine:3.17 tmpimg + run ch-image modify --ci-debug -S "doesnotexist" -- alpine:3.17 tmpimg echo "$output" [[ $status -eq 1 ]] [[ $output = *'Unable to run shell:'* ]] From 6bc35a9b1105fc59de41d707f33623519fd8221c Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Thu, 6 Jun 2024 17:22:21 +0000 Subject: [PATCH 39/65] skip mysteriously failing test for now --- test/force-auto.py.in | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/force-auto.py.in b/test/force-auto.py.in index 9bfab7a39..a74d4783f 100644 --- a/test/force-auto.py.in +++ b/test/force-auto.py.in @@ -121,6 +121,11 @@ class Test(abc.ABC): if (self.force in self.force_excludes): print(f"\n# skip: {self}: --force=%s excluded" % self.force) return + # targeted skip, see issue #1904 + if (self.base == "quay.io/centos/centos:stream8" and self.run == Run.NEEDED and self.force == "seccomp" and not self.preprep): + skip = "skip 'see issue #1904'" + else: + skip = "" # scope if ( (self.scope == Scope.STANDARD or self.run == Run.NEEDED) and self.force != "fakeroot"): @@ -168,6 +173,7 @@ echo "$output" # emit the test print(f""" @test "ch-image --force: {self}" {{ +{skip} scope {scope} {arch_excludes} From 8a97d8dc2dbb94d6179d1ef80cff48ed8ec152d9 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Fri, 7 Jun 2024 16:52:42 +0000 Subject: [PATCH 40/65] work on some suggestions --- bin/ch-image.py.in | 7 +++-- bin/ch_fuse.c | 2 +- lib/build.py | 59 +++++++++++++++++++------------------ lib/charliecloud.py | 6 ++++ lib/pull.py | 2 +- test/build/50_ch-image.bats | 4 +-- test/run/ch-run_misc.bats | 3 -- 7 files changed, 46 insertions(+), 37 deletions(-) diff --git a/bin/ch-image.py.in b/bin/ch-image.py.in index a34f1e457..70351e338 100644 --- a/bin/ch-image.py.in +++ b/bin/ch-image.py.in @@ -284,8 +284,11 @@ def main(): add_opts(sp, build.modify, deps_check=True, stog_init=True) sp.add_argument("-c", metavar="CMD", action="append", default=[], nargs=1, help="foo") sp.add_argument("-S", "--shell", metavar="shell", help="foo") - # FIXME: rename - sp.add_argument("--ci-debug", action="store_true", help="CI debugging, remove this before merge!") + # “--ci-automated” is a hidden option for automated CI pipelines (e.g. Github + # Actions). The absence of an interactive terminal in such pipelines can + # break the internal logic of “modify”, which generally checks for an + # interactive shell when determining what to do. + sp.add_argument("--ci-automated", action="store_true", help=argparse.SUPPRESS) sp.add_argument("image_ref", metavar="IMAGE_REF", help="image to modify") sp.add_argument("out_image", metavar="OUT_IMAGE", help="destination of modified image") # Optional positional argument? https://stackoverflow.com/a/4480202 diff --git a/bin/ch_fuse.c b/bin/ch_fuse.c index 0e1fdb19e..e8cbcbc55 100644 --- a/bin/ch_fuse.c +++ b/bin/ch_fuse.c @@ -204,7 +204,7 @@ int sq_loop(void) // [1]: https://codereview.stackexchange.com/a/109349 // [2]: https://man7.org/linux/man-pages/man2/wait.2.html exit_code = 128 + WTERMSIG(child_status); - VERBOSE("child terminated by signal %d", exit_code - 128) + VERBOSE("child terminated by signal %d", WTERMSIG(child_status)) } } diff --git a/lib/build.py b/lib/build.py index 1992ee822..b55e2b713 100644 --- a/lib/build.py +++ b/lib/build.py @@ -274,7 +274,8 @@ def modify(cli_): cli = cli_ # This file assumes that global cli comes from the “build” function. If we - # don’t assign these values, it casues problems. + # don’t assign these values, the program will fail after trying to access + # them. cli.parse_only = False cli.force = ch.Force_Mode.SECCOMP cli.force_cmd = force.FORCE_CMD_DEFAULT @@ -296,24 +297,15 @@ def modify(cli_): ch.FATAL("%s: no such file" % cli.script) # This kludge is necessary because cli is a global variable, with cli.tag - # assumed present elsewhere in the file. cli.tag represents the image being - # built, which in our case can either be the source image or the output image - # (if specified). + # assumed present elsewhere in the file. Here, cli.tag represents the + # destination image. cli.tag = str(out_image) - ch.ILLERI(cli.script) - - if ((not sys.stdin.isatty()) and (commands == []) and (not cli.ci_debug)): + if ((not sys.stdin.isatty()) and (commands == []) and (not cli.ci_automated)): # https://stackoverflow.com/a/6482200 stdin = sys.stdin.read() - - - # FIXME: remove - if (cli.ci_debug): - ch.INFO("STDIN!!!") - ch.INFO(stdin) # We use “decode("utf-8")” here because stdout seems default to a bytes # object, which is not a valid type for an argument for “Path”. tmpfile = ch.Path(subprocess.run(["mktemp", "-d"],capture_output=True).stdout.decode("utf-8")) @@ -328,8 +320,9 @@ def modify(cli_): shell = cli.shell else: shell = "/bin/sh" + # FIXME: This logic is a bit busted if ((commands not in [[],['']]) or (cli.script is not None)): - # FIXME: Kludge!!! + # non-interactive case (“-c” or script) if (cli.script is not None): tree = modify_tree_make_script(src_image.ref, cli.script) else: @@ -341,6 +334,8 @@ def modify(cli_): ml = traverse_parse_tree(tree) else: + # Interactive case + # Generate “fake” SID for build cache. We do this because we can’t compute # an SID, but we still want to make sure that it’s unique enough that # we’re unlikely to run into a collision. @@ -352,9 +347,7 @@ def modify(cli_): bu.cache.branch_nocheckout(src_image.ref, out_image.ref) foo = subprocess.run([ch.CH_BIN + "/ch-run", "--unsafe", "-w", str(out_image.ref), "--", shell]) - # FIXME: This causes issues when you change the value in ch_misc.h and - # forget to change it here... - if (foo.returncode == 49): + if (foo.returncode == ch.Ch_Run_Retcode.ERR_CMD.value): # FIXME: Write a better error message? ch.FATAL("Unable to run shell: %s" % shell) ch.VERBOSE("using SID %s" % fake_sid) @@ -397,14 +390,17 @@ def modify_tree_make(src_img, cmds): # attribute a debug value of -1 to avoid said errors. meta = lark.tree.Meta() meta.line = -1 - #df_children.append(im.Tree(lark.Token('RULE', 'from_'), [im.Tree(lark.Token('RULE', 'image_ref'),[lark.Token('IMAGE_REF', src_img.name)], meta)], meta)) - df_children.append(im.Tree(lark.Token('RULE', 'from_'), [im.Tree(lark.Token('RULE', 'image_ref'),[lark.Token('IMAGE_REF', str(src_img))], meta)], meta)) + df_children.append(im.Tree(lark.Token('RULE', 'from_'), + [im.Tree(lark.Token('RULE', 'image_ref'), + [lark.Token('IMAGE_REF', str(src_img))], meta)], meta)) if (cli.shell is not None): - #df_children.append(im.Tree(lark.Token('RULE', 'shell'), [lark.Token('STRING_QUO', "/bin/sh"),lark.Token('STRING_QUO', "-c")], meta)) - #ch.ILLERI("HERE") - df_children.append(im.Tree(lark.Token('RULE', 'shell'), [lark.Token('STRING_QUOTED', '"%s"' % cli.shell),lark.Token('STRING_QUOTED', '"-c"')],meta)) + df_children.append(im.Tree(lark.Token('RULE', 'shell'), + [lark.Token('STRING_QUOTED', '"%s"' % cli.shell), + lark.Token('STRING_QUOTED', '"-c"')],meta)) for cmd in cmds: - df_children.append(im.Tree(lark.Token('RULE', 'run'), [im.Tree(lark.Token('RULE', 'run_shell'),[lark.Token('LINE_CHUNK', cmd)], meta)],meta)) + df_children.append(im.Tree(lark.Token('RULE', 'run'), + [im.Tree(lark.Token('RULE', 'run_shell'), + [lark.Token('LINE_CHUNK', cmd)], meta)],meta)) return im.Tree(lark.Token('RULE', 'start'), [im.Tree(lark.Token('RULE','dockerfile'), df_children)], meta) # FIXME: Probably should merge into “modify_tree_make” @@ -438,12 +434,19 @@ def modify_tree_make_script(src_img, path): # attribute a debug value of -1 to avoid said errors. meta = lark.tree.Meta() meta.line = -1 - df_children.append(im.Tree(lark.Token('RULE', 'from_'), [im.Tree(lark.Token('RULE', 'image_ref'),[lark.Token('IMAGE_REF', str(src_img))], meta)], meta)) + df_children.append(im.Tree(lark.Token('RULE', 'from_'), + [im.Tree(lark.Token('RULE', 'image_ref'), + [lark.Token('IMAGE_REF', str(src_img))], meta)], meta)) if (cli.shell is not None): - #df_children.append(im.Tree(lark.Token('RULE', 'from_'), [im.Tree(lark.Token('RULE', 'shell'),[lark.Token('STRING_QUO', "%s" % cli.shell),lark.Token('STRING_QUO', "-c")], meta)], meta)) - df_children.append(im.Tree(lark.Token('RULE', 'shell'), [lark.Token('STRING_QUOTED', '"%s"' % cli.shell),lark.Token('STRING_QUOTED', '"-c"')],meta)) - df_children.append(im.Tree(lark.Token('RULE', 'copy'), [im.Tree(lark.Token('RULE', 'copy_shell'),[lark.Token('WORD', path),lark.Token('WORD', '/ch/script.sh')], meta)],meta)) - df_children.append(im.Tree(lark.Token('RULE', 'run'), [im.Tree(lark.Token('RULE', 'run_shell'),[lark.Token('LINE_CHUNK', '/ch/script.sh')], meta)],meta)) + df_children.append(im.Tree(lark.Token('RULE', 'shell'), + [lark.Token('STRING_QUOTED', '"%s"' % cli.shell), + lark.Token('STRING_QUOTED', '"-c"')],meta)) + df_children.append(im.Tree(lark.Token('RULE', 'copy'), + [im.Tree(lark.Token('RULE', 'copy_shell'),[lark.Token('WORD', path), + lark.Token('WORD', '/ch/script.sh')], meta)],meta)) + df_children.append(im.Tree(lark.Token('RULE', 'run'), + [im.Tree(lark.Token('RULE', 'run_shell'), + [lark.Token('LINE_CHUNK', '/ch/script.sh')], meta)],meta)) return im.Tree(lark.Token('RULE', 'start'), [im.Tree(lark.Token('RULE','dockerfile'), df_children)], meta) # Traverse Lark parse tree and do what it says. diff --git a/lib/charliecloud.py b/lib/charliecloud.py index 988a4054c..29de18ca8 100644 --- a/lib/charliecloud.py +++ b/lib/charliecloud.py @@ -41,6 +41,12 @@ class Build_Mode(enum.Enum): DISABLED = "disabled" REBUILD = "rebuild" +# ch-run exit codes (see also: bin/ch_misc.h) +class Ch_Run_Retcode(enum.Enum): + ERR_CHRUN = 31 + ERR_CMD = 49 + ERR_SQUASH = 84 + # Download cache mode. class Download_Mode(enum.Enum): ENABLED = "enabled" diff --git a/lib/pull.py b/lib/pull.py index 24e1714b2..cea1ec842 100644 --- a/lib/pull.py +++ b/lib/pull.py @@ -106,7 +106,7 @@ def download(self): # currently, this error is only raised if we’ve downloaded the # skinny manifest. have_skinny = True - if (ch.arch == "amd64"): + if (ch.arch in ["amd64", "aarch64"]): # We’re guessing that enough arch-unaware images are amd64 to # barge ahead if requested architecture is amd64. ch.arch = "yolo" diff --git a/test/build/50_ch-image.bats b/test/build/50_ch-image.bats index bec216461..806e4b3ac 100644 --- a/test/build/50_ch-image.bats +++ b/test/build/50_ch-image.bats @@ -999,7 +999,7 @@ EOF # non-interactive, script echo "touch /home/bar" >> "${BATS_TMPDIR}/modify-script.sh" chmod 755 "${BATS_TMPDIR}/modify-script.sh" - ch-image modify --ci-debug alpine:3.17 tmpimg "${BATS_TMPDIR}/modify-script.sh" + ch-image modify --ci-automated alpine:3.17 tmpimg "${BATS_TMPDIR}/modify-script.sh" run ch-run tmpimg -- ls /home echo "$output" [[ $status -eq 0 ]] @@ -1021,7 +1021,7 @@ EOF [[ $output = *'output must be different from source image'* ]] # non-existant shell - run ch-image modify --ci-debug -S "doesnotexist" -- alpine:3.17 tmpimg + run ch-image modify --ci-automated -S "doesnotexist" -- alpine:3.17 tmpimg echo "$output" [[ $status -eq 1 ]] [[ $output = *'Unable to run shell:'* ]] diff --git a/test/run/ch-run_misc.bats b/test/run/ch-run_misc.bats index 157695785..53a6129e9 100644 --- a/test/run/ch-run_misc.bats +++ b/test/run/ch-run_misc.bats @@ -316,9 +316,6 @@ EOF # no argument to --bind run ch-run "$ch_timg" -b echo "$output" - echo "STATUS" - echo "$status" - echo "STATUS" [[ $status -eq 64 ]] [[ $output = *'option requires an argument'* ]] From 5be8c3e5a251e3701eea6a0cde1726d0bfaf854b Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Fri, 7 Jun 2024 18:07:15 +0000 Subject: [PATCH 41/65] more RPM issues? --- test/force-auto.py.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/force-auto.py.in b/test/force-auto.py.in index a74d4783f..60575e4c1 100644 --- a/test/force-auto.py.in +++ b/test/force-auto.py.in @@ -122,7 +122,8 @@ class Test(abc.ABC): print(f"\n# skip: {self}: --force=%s excluded" % self.force) return # targeted skip, see issue #1904 - if (self.base == "quay.io/centos/centos:stream8" and self.run == Run.NEEDED and self.force == "seccomp" and not self.preprep): + if ((self.base == "quay.io/centos/centos:stream8" and self.run == Run.NEEDED and self.force == "seccomp" and not self.preprep) + or (self.base == "fedora:26" and self.run == Run.NEEDED and self.force == "fakeroot" and not self.preprep)): skip = "skip 'see issue #1904'" else: skip = "" From c53a82082e4b3f7a32e2bc7a6abe0eff7659fb3f Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Fri, 7 Jun 2024 18:17:14 +0000 Subject: [PATCH 42/65] oops, misread failure --- test/force-auto.py.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/force-auto.py.in b/test/force-auto.py.in index 60575e4c1..b1ef7ec4b 100644 --- a/test/force-auto.py.in +++ b/test/force-auto.py.in @@ -123,7 +123,7 @@ class Test(abc.ABC): return # targeted skip, see issue #1904 if ((self.base == "quay.io/centos/centos:stream8" and self.run == Run.NEEDED and self.force == "seccomp" and not self.preprep) - or (self.base == "fedora:26" and self.run == Run.NEEDED and self.force == "fakeroot" and not self.preprep)): + or (self.base == "fedora:26" and self.run == Run.NEEDED and self.force == "seccomp" and not self.preprep)): skip = "skip 'see issue #1904'" else: skip = "" From 1dcbb4db17ef0c28d592de639dbaffedc3b515a5 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Fri, 7 Jun 2024 21:00:44 +0000 Subject: [PATCH 43/65] what happens when I do this --- lib/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/build.py b/lib/build.py index b55e2b713..9d498fb25 100644 --- a/lib/build.py +++ b/lib/build.py @@ -321,7 +321,7 @@ def modify(cli_): else: shell = "/bin/sh" # FIXME: This logic is a bit busted - if ((commands not in [[],['']]) or (cli.script is not None)): + if ((commands != []) or (cli.script is not None)): # non-interactive case (“-c” or script) if (cli.script is not None): tree = modify_tree_make_script(src_image.ref, cli.script) From 3314546c79ddd877cafcfbdd71b68a857e8e1712 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Mon, 10 Jun 2024 18:45:51 +0000 Subject: [PATCH 44/65] try removing hidden ci opt --- lib/build.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/build.py b/lib/build.py index 9d498fb25..d26b05689 100644 --- a/lib/build.py +++ b/lib/build.py @@ -301,11 +301,12 @@ def modify(cli_): # destination image. cli.tag = str(out_image) + stdin = sys.stdin.read() - if ((not sys.stdin.isatty()) and (commands == []) and (not cli.ci_automated)): + # FIXME: If this passes CI, try removing empty string from last check + if ((not sys.stdin.isatty()) and (commands == []) and (stdin not in [None, ''])): # https://stackoverflow.com/a/6482200 - stdin = sys.stdin.read() # We use “decode("utf-8")” here because stdout seems default to a bytes # object, which is not a valid type for an argument for “Path”. tmpfile = ch.Path(subprocess.run(["mktemp", "-d"],capture_output=True).stdout.decode("utf-8")) From 6ec4b265c4930b79326f44f5063ddcc066c3058e Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Mon, 10 Jun 2024 19:22:28 +0000 Subject: [PATCH 45/65] fully remove automated ci option --- bin/ch-image.py.in | 5 ----- lib/build.py | 3 ++- test/build/50_ch-image.bats | 4 ++-- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/bin/ch-image.py.in b/bin/ch-image.py.in index 70351e338..31227d783 100644 --- a/bin/ch-image.py.in +++ b/bin/ch-image.py.in @@ -284,11 +284,6 @@ def main(): add_opts(sp, build.modify, deps_check=True, stog_init=True) sp.add_argument("-c", metavar="CMD", action="append", default=[], nargs=1, help="foo") sp.add_argument("-S", "--shell", metavar="shell", help="foo") - # “--ci-automated” is a hidden option for automated CI pipelines (e.g. Github - # Actions). The absence of an interactive terminal in such pipelines can - # break the internal logic of “modify”, which generally checks for an - # interactive shell when determining what to do. - sp.add_argument("--ci-automated", action="store_true", help=argparse.SUPPRESS) sp.add_argument("image_ref", metavar="IMAGE_REF", help="image to modify") sp.add_argument("out_image", metavar="OUT_IMAGE", help="destination of modified image") # Optional positional argument? https://stackoverflow.com/a/4480202 diff --git a/lib/build.py b/lib/build.py index d26b05689..cd4bb31dc 100644 --- a/lib/build.py +++ b/lib/build.py @@ -303,7 +303,8 @@ def modify(cli_): stdin = sys.stdin.read() - # FIXME: If this passes CI, try removing empty string from last check + # We check that stdin isn’t None to ensure that we don’t go down this code + # path by mistake. if ((not sys.stdin.isatty()) and (commands == []) and (stdin not in [None, ''])): # https://stackoverflow.com/a/6482200 diff --git a/test/build/50_ch-image.bats b/test/build/50_ch-image.bats index 806e4b3ac..389802ab9 100644 --- a/test/build/50_ch-image.bats +++ b/test/build/50_ch-image.bats @@ -999,7 +999,7 @@ EOF # non-interactive, script echo "touch /home/bar" >> "${BATS_TMPDIR}/modify-script.sh" chmod 755 "${BATS_TMPDIR}/modify-script.sh" - ch-image modify --ci-automated alpine:3.17 tmpimg "${BATS_TMPDIR}/modify-script.sh" + ch-image modify alpine:3.17 tmpimg "${BATS_TMPDIR}/modify-script.sh" run ch-run tmpimg -- ls /home echo "$output" [[ $status -eq 0 ]] @@ -1021,7 +1021,7 @@ EOF [[ $output = *'output must be different from source image'* ]] # non-existant shell - run ch-image modify --ci-automated -S "doesnotexist" -- alpine:3.17 tmpimg + run ch-image modify -S "doesnotexist" -- alpine:3.17 tmpimg echo "$output" [[ $status -eq 1 ]] [[ $output = *'Unable to run shell:'* ]] From b6da358f93a6386ad6d9018fda84b9f0543339f8 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Mon, 10 Jun 2024 21:38:22 +0000 Subject: [PATCH 46/65] hopefully this also works in CI --- lib/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/build.py b/lib/build.py index cd4bb31dc..e1729c93f 100644 --- a/lib/build.py +++ b/lib/build.py @@ -304,8 +304,8 @@ def modify(cli_): stdin = sys.stdin.read() # We check that stdin isn’t None to ensure that we don’t go down this code - # path by mistake. - if ((not sys.stdin.isatty()) and (commands == []) and (stdin not in [None, ''])): + # path by mistake (e.g. in CI, where stdin will never by a TTY). + if ((not sys.stdin.isatty()) and (commands == []) and (stdin not in None)): # https://stackoverflow.com/a/6482200 # We use “decode("utf-8")” here because stdout seems default to a bytes From 2b96eccbbc92186a81d73482a9d845cb4a3189da Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Mon, 10 Jun 2024 21:48:30 +0000 Subject: [PATCH 47/65] fix syntax error --- lib/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/build.py b/lib/build.py index e1729c93f..4129162fb 100644 --- a/lib/build.py +++ b/lib/build.py @@ -305,7 +305,7 @@ def modify(cli_): # We check that stdin isn’t None to ensure that we don’t go down this code # path by mistake (e.g. in CI, where stdin will never by a TTY). - if ((not sys.stdin.isatty()) and (commands == []) and (stdin not in None)): + if ((not sys.stdin.isatty()) and (commands == []) and (stdin != None)): # https://stackoverflow.com/a/6482200 # We use “decode("utf-8")” here because stdout seems default to a bytes From ed46d89d239277cdf32e9cb54bc81a602658d7d3 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Mon, 10 Jun 2024 22:32:40 +0000 Subject: [PATCH 48/65] check for empty string instead of None --- lib/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/build.py b/lib/build.py index 4129162fb..cb2d6d07e 100644 --- a/lib/build.py +++ b/lib/build.py @@ -305,7 +305,7 @@ def modify(cli_): # We check that stdin isn’t None to ensure that we don’t go down this code # path by mistake (e.g. in CI, where stdin will never by a TTY). - if ((not sys.stdin.isatty()) and (commands == []) and (stdin != None)): + if ((not sys.stdin.isatty()) and (commands == []) and (stdin != '')): # https://stackoverflow.com/a/6482200 # We use “decode("utf-8")” here because stdout seems default to a bytes From fc5c89a77efc2c926f5e1fdbd4359604ee39fd82 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Tue, 11 Jun 2024 18:47:06 +0000 Subject: [PATCH 49/65] update comment [skip ci] --- lib/build.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/build.py b/lib/build.py index cb2d6d07e..7988eca14 100644 --- a/lib/build.py +++ b/lib/build.py @@ -405,8 +405,7 @@ def modify_tree_make(src_img, cmds): [lark.Token('LINE_CHUNK', cmd)], meta)],meta)) return im.Tree(lark.Token('RULE', 'start'), [im.Tree(lark.Token('RULE','dockerfile'), df_children)], meta) -# FIXME: Probably should merge into “modify_tree_make” -# (This is a tmp function to implement functionality) +# FIXME: Combine with “modify_tree_make”? def modify_tree_make_script(src_img, path): """Temporary(?) analog of “modify_tree_make” for the non-interactive version of “modify” using a script. For the command line: From dc5ed4db62e0f5e90b80602b618308a9d4b075ce Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Fri, 21 Jun 2024 18:56:19 +0000 Subject: [PATCH 50/65] fix interactive case --- lib/build.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/lib/build.py b/lib/build.py index 7988eca14..8dfcd6d95 100644 --- a/lib/build.py +++ b/lib/build.py @@ -301,22 +301,24 @@ def modify(cli_): # destination image. cli.tag = str(out_image) - stdin = sys.stdin.read() # We check that stdin isn’t None to ensure that we don’t go down this code # path by mistake (e.g. in CI, where stdin will never by a TTY). - if ((not sys.stdin.isatty()) and (commands == []) and (stdin != '')): - # https://stackoverflow.com/a/6482200 - - # We use “decode("utf-8")” here because stdout seems default to a bytes - # object, which is not a valid type for an argument for “Path”. - tmpfile = ch.Path(subprocess.run(["mktemp", "-d"],capture_output=True).stdout.decode("utf-8")) - with open(tmpfile, "w") as outfile: - outfile.write(stdin) - # By default, the file is seemingly created with its execute bit - # unassigned. This is problematic for the RUN instruction. - os.chmod(tmpfile, 0o755) - cli.script = str(tmpfile) + if ((not sys.stdin.isatty()) and (commands == [])): + stdin = sys.stdin.read() + + if (stdin != ''): + # https://stackoverflow.com/a/6482200 + + # We use “decode("utf-8")” here because stdout seems default to a bytes + # object, which is not a valid type for an argument for “Path”. + tmpfile = ch.Path(subprocess.run(["mktemp", "-d"],capture_output=True).stdout.decode("utf-8")) + with open(tmpfile, "w") as outfile: + outfile.write(stdin) + # By default, the file is seemingly created with its execute bit + # unassigned. This is problematic for the RUN instruction. + os.chmod(tmpfile, 0o755) + cli.script = str(tmpfile) if (cli.shell is not None): shell = cli.shell From 7c320e7d5f142c7c1781ee1c867a7b79bab3ce93 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Fri, 5 Jul 2024 14:43:02 +0000 Subject: [PATCH 51/65] create new Python module --- bin/ch-image.py.in | 3 +- lib/build.py | 254 ++++++++------------------------------------- lib/modify.py | 204 ++++++++++++++++++++++++++++++++++++ 3 files changed, 250 insertions(+), 211 deletions(-) create mode 100644 lib/modify.py diff --git a/bin/ch-image.py.in b/bin/ch-image.py.in index 31227d783..61802cbc2 100644 --- a/bin/ch-image.py.in +++ b/bin/ch-image.py.in @@ -15,6 +15,7 @@ import build_cache as bu import filesystem as fs import image as im import misc +import modify import pull import push @@ -281,7 +282,7 @@ def main(): # modify sp = ap.add_parser("modify", "foo") - add_opts(sp, build.modify, deps_check=True, stog_init=True) + add_opts(sp, modify.main, deps_check=True, stog_init=True) sp.add_argument("-c", metavar="CMD", action="append", default=[], nargs=1, help="foo") sp.add_argument("-S", "--shell", metavar="shell", help="foo") sp.add_argument("image_ref", metavar="IMAGE_REF", help="image to modify") diff --git a/lib/build.py b/lib/build.py index 8dfcd6d95..28eae8afd 100644 --- a/lib/build.py +++ b/lib/build.py @@ -11,7 +11,6 @@ import shutil import subprocess import sys -import uuid import charliecloud as ch import build_cache as bu @@ -122,13 +121,29 @@ def __default__(self, tree): self.inst_prev = inst self.instruction_total_ct += 1 - def main(cli_): # CLI namespace. :P global cli cli = cli_ + # Process CLI. Make appropriate modifications to “cli” instance and return + # Dockerfile text. + text = cli_process(cli) + + tree = parse_dockerfile(text) + + global image_ct + ml = traverse_parse_tree(tree, image_ct, cli) + +## Functions ## + +# Function that processes parsed CLI, modifying the passed “cli” object +# appropriatley as it does. Returns the text of the file used for the build +# operation. Note that Python passes variables to functions by their object +# reference, so changes made to mutable objects (which “cli” is) will persist in +# the scope of the caller.' +def cli_process(cli): # Infer input file if needed. if (cli.file is None): cli.file = cli.context + "/Dockerfile" @@ -215,6 +230,9 @@ def build_arg_get(arg): text = ch.ossafe("can’t read: %s" % cli.file, fp.read) ch.close_(fp) + return text + +def parse_dockerfile(text): # Parse it. parser = lark.Lark(im.GRAMMAR_DOCKERFILE, parser="earley", propagate_positions=True, tree_class=im.Tree) @@ -245,212 +263,7 @@ def build_arg_get(arg): ch.ERROR("Dockerfile uses RSYNC, so rsync(1) is required") raise - ml = traverse_parse_tree(tree) - - # Check that all build arguments were consumed. - if (len(cli.build_arg) != 0): - ch.FATAL("--build-arg: not consumed: " + " ".join(cli.build_arg.keys())) - - # Print summary & we’re done. - if (ml.instruction_total_ct == 0): - ch.FATAL("no instructions found: %s" % cli.file) - assert (ml.inst_prev.image_i + 1 == image_ct) # should’ve errored already - if ((cli.force != ch.Force_Mode.NONE) and ml.miss_ct != 0): - ch.INFO("--force=%s: modified %d RUN instructions" - % (cli.force.value, forcer.run_modified_ct)) - ch.INFO("grown in %d instructions: %s" - % (ml.instruction_total_ct, ml.inst_prev.image)) - # FIXME: remove when we’re done encouraging people to use the build cache. - if (isinstance(bu.cache, bu.Disabled_Cache)): - ch.INFO("build slow? consider enabling the build cache", - "https://hpc.github.io/charliecloud/command-usage.html#build-cache") - - -## Functions ## - -def modify(cli_): - # In this file, “cli” is used as a global variable - global cli - cli = cli_ - - # This file assumes that global cli comes from the “build” function. If we - # don’t assign these values, the program will fail after trying to access - # them. - cli.parse_only = False - cli.force = ch.Force_Mode.SECCOMP - cli.force_cmd = force.FORCE_CMD_DEFAULT - cli.bind = [] - cli.context = os.path.abspath(os.sep) - - commands = [] - # “Flatten” commands array - for c in cli.c: - commands += c - src_image = im.Image(im.Reference(cli.image_ref)) - out_image = im.Image(im.Reference(cli.out_image)) - if (not src_image.unpack_exist_p): - ch.FATAL("not in storage: %s" % src_image.ref) - if (cli.out_image == cli.image_ref): - ch.FATAL("output must be different from source image (%s)" % cli.image_ref) - if (cli.script is not None): - if (not ch.Path(cli.script).exists): - ch.FATAL("%s: no such file" % cli.script) - - # This kludge is necessary because cli is a global variable, with cli.tag - # assumed present elsewhere in the file. Here, cli.tag represents the - # destination image. - cli.tag = str(out_image) - - - # We check that stdin isn’t None to ensure that we don’t go down this code - # path by mistake (e.g. in CI, where stdin will never by a TTY). - if ((not sys.stdin.isatty()) and (commands == [])): - stdin = sys.stdin.read() - - if (stdin != ''): - # https://stackoverflow.com/a/6482200 - - # We use “decode("utf-8")” here because stdout seems default to a bytes - # object, which is not a valid type for an argument for “Path”. - tmpfile = ch.Path(subprocess.run(["mktemp", "-d"],capture_output=True).stdout.decode("utf-8")) - with open(tmpfile, "w") as outfile: - outfile.write(stdin) - # By default, the file is seemingly created with its execute bit - # unassigned. This is problematic for the RUN instruction. - os.chmod(tmpfile, 0o755) - cli.script = str(tmpfile) - - if (cli.shell is not None): - shell = cli.shell - else: - shell = "/bin/sh" - # FIXME: This logic is a bit busted - if ((commands != []) or (cli.script is not None)): - # non-interactive case (“-c” or script) - if (cli.script is not None): - tree = modify_tree_make_script(src_image.ref, cli.script) - else: - tree = modify_tree_make(src_image.ref, commands) - - # Count the number of stages (i.e., FROM instructions) - global image_ct - image_ct = sum(1 for i in tree.children_("from_")) - - ml = traverse_parse_tree(tree) - else: - # Interactive case - - # Generate “fake” SID for build cache. We do this because we can’t compute - # an SID, but we still want to make sure that it’s unique enough that - # we’re unlikely to run into a collision. - fake_sid = uuid.uuid4() - out_image.unpack_clear() - out_image.copy_unpacked(src_image) - bu.cache.worktree_adopt(out_image, src_image.ref.for_path) - bu.cache.ready(out_image) - bu.cache.branch_nocheckout(src_image.ref, out_image.ref) - foo = subprocess.run([ch.CH_BIN + "/ch-run", "--unsafe", "-w", - str(out_image.ref), "--", shell]) - if (foo.returncode == ch.Ch_Run_Retcode.ERR_CMD.value): - # FIXME: Write a better error message? - ch.FATAL("Unable to run shell: %s" % shell) - ch.VERBOSE("using SID %s" % fake_sid) - # FIXME: metadata history stuff? See misc.import_. - if (out_image.metadata["history"] == []): - out_image.metadata["history"].append({ "empty_layer": False, - "command": "ch-image import"}) - out_image.metadata_save() - bu.cache.commit(out_image.unpack_path, fake_sid, "MODIFY interactive", []) - -def modify_tree_make(src_img, cmds): - """Function that manually constructs a parse tree corresponding to a set of - “ch-image modify” commands, as though the commands had been specified in a - Dockerfile. Note that because “ch-image modify” simply executes one or - more commands inside a container, the only Dockerfile instructions we need - to consider are “FROM” and “RUN”. E.g. for the command line - - $ ch-image modify -c 'echo foo' -c 'echo bar' -- foo foo2 - - this function produces the following parse tree - - start - dockerfile - from_ - image_ref - IMAGE_REF foo - run - run_shell - LINE_CHUNK echo foo - run - run_shell - LINE_CHUNK echo bar - """ - # Children of dockerfile tree - df_children = [] - # Metadata attribute. We use this attribute in the “_pretty” method for our - # “Tree” class. Constructing a tree without specifying a “Meta” instance that - # has been given a “line” value will result in the attribute not being present, - # which causes an error when we try to access that attribute. Here we give the - # attribute a debug value of -1 to avoid said errors. - meta = lark.tree.Meta() - meta.line = -1 - df_children.append(im.Tree(lark.Token('RULE', 'from_'), - [im.Tree(lark.Token('RULE', 'image_ref'), - [lark.Token('IMAGE_REF', str(src_img))], meta)], meta)) - if (cli.shell is not None): - df_children.append(im.Tree(lark.Token('RULE', 'shell'), - [lark.Token('STRING_QUOTED', '"%s"' % cli.shell), - lark.Token('STRING_QUOTED', '"-c"')],meta)) - for cmd in cmds: - df_children.append(im.Tree(lark.Token('RULE', 'run'), - [im.Tree(lark.Token('RULE', 'run_shell'), - [lark.Token('LINE_CHUNK', cmd)], meta)],meta)) - return im.Tree(lark.Token('RULE', 'start'), [im.Tree(lark.Token('RULE','dockerfile'), df_children)], meta) - -# FIXME: Combine with “modify_tree_make”? -def modify_tree_make_script(src_img, path): - """Temporary(?) analog of “modify_tree_make” for the non-interactive version - of “modify” using a script. For the command line: - - $ ch-image modify foo foo2 /path/to/script - - this function produces the following parse tree - - start - dockerfile - from_ - image_ref - IMAGE_REF foo - copy - copy_shell - WORD /path/to/script WORD /ch/script.sh - run - run_shell - LINE_CHUNK /ch/script.sh - """ - # Children of dockerfile tree - df_children = [] - # Metadata attribute. We use this attribute in the “_pretty” method for our - # “Tree” class. Constructing a tree without specifying a “Meta” instance that - # has been given a “line” value will result in the attribute not being present, - # which causes an error when we try to access that attribute. Here we give the - # attribute a debug value of -1 to avoid said errors. - meta = lark.tree.Meta() - meta.line = -1 - df_children.append(im.Tree(lark.Token('RULE', 'from_'), - [im.Tree(lark.Token('RULE', 'image_ref'), - [lark.Token('IMAGE_REF', str(src_img))], meta)], meta)) - if (cli.shell is not None): - df_children.append(im.Tree(lark.Token('RULE', 'shell'), - [lark.Token('STRING_QUOTED', '"%s"' % cli.shell), - lark.Token('STRING_QUOTED', '"-c"')],meta)) - df_children.append(im.Tree(lark.Token('RULE', 'copy'), - [im.Tree(lark.Token('RULE', 'copy_shell'),[lark.Token('WORD', path), - lark.Token('WORD', '/ch/script.sh')], meta)],meta)) - df_children.append(im.Tree(lark.Token('RULE', 'run'), - [im.Tree(lark.Token('RULE', 'run_shell'), - [lark.Token('LINE_CHUNK', '/ch/script.sh')], meta)],meta)) - return im.Tree(lark.Token('RULE', 'start'), [im.Tree(lark.Token('RULE','dockerfile'), df_children)], meta) + return tree # Traverse Lark parse tree and do what it says. # @@ -467,7 +280,11 @@ def modify_tree_make_script(src_img, path): # [1]: https://lark-parser.readthedocs.io/en/latest/visitors/#visitors # [2]: https://github.com/lark-parser/lark/blob/445c8d4/lark/visitors.py#L211 # [3]: https://lark-parser.readthedocs.io/en/latest/classes/#tree -def traverse_parse_tree(tree): +def traverse_parse_tree(tree, image_ct_, cli_): + global cli + global image_ct + cli = cli_ + image_ct = image_ct_ ml = Main_Loop() if (hasattr(ml, 'visit_topdown')): ml.visit_topdown(tree) @@ -477,7 +294,24 @@ def traverse_parse_tree(tree): if (ml.miss_ct == 0): ml.inst_prev.checkout() ml.inst_prev.ready() - return ml + + # Check that all build arguments were consumed. + if (len(cli.build_arg) != 0): + ch.FATAL("--build-arg: not consumed: " + " ".join(cli.build_arg.keys())) + + # Print summary & we’re done. + if (ml.instruction_total_ct == 0): + ch.FATAL("no instructions found: %s" % cli.file) + assert (ml.inst_prev.image_i + 1 == image_ct) # should’ve errored already + if ((cli.force != ch.Force_Mode.NONE) and ml.miss_ct != 0): + ch.INFO("--force=%s: modified %d RUN instructions" + % (cli.force.value, forcer.run_modified_ct)) + ch.INFO("grown in %d instructions: %s" + % (ml.instruction_total_ct, ml.inst_prev.image)) + # FIXME: remove when we’re done encouraging people to use the build cache. + if (isinstance(bu.cache, bu.Disabled_Cache)): + ch.INFO("build slow? consider enabling the build cache", + "https://hpc.github.io/charliecloud/command-usage.html#build-cache") def unescape(sl): # FIXME: This is also ugly and should go in the grammar. diff --git a/lib/modify.py b/lib/modify.py new file mode 100644 index 000000000..9f985b661 --- /dev/null +++ b/lib/modify.py @@ -0,0 +1,204 @@ +# implementation of ch-image modify + +import os +import subprocess +import sys +import uuid + +import charliecloud as ch +import build +import build_cache as bu +import force +import image as im + +lark = im.lark + +def main(cli_): + global called + called = True + + # Need to pass tree to build.py + global tree + + # In this file, “cli” is used as a global variable + global cli + cli = cli_ + + # This file assumes that global cli comes from the “build” function. If we + # don’t assign these values, the program will fail after trying to access + # them. FIXME: Can partially fix this by adding command line opts. + cli.parse_only = False + cli.force = ch.Force_Mode.SECCOMP + cli.force_cmd = force.FORCE_CMD_DEFAULT + cli.bind = [] + cli.build_arg = [] + cli.context = os.path.abspath(os.sep) + + commands = [] + # “Flatten” commands array + for c in cli.c: + commands += c + src_image = im.Image(im.Reference(cli.image_ref)) + out_image = im.Image(im.Reference(cli.out_image)) + if (not src_image.unpack_exist_p): + ch.FATAL("not in storage: %s" % src_image.ref) + if (cli.out_image == cli.image_ref): + ch.FATAL("output must be different from source image (%s)" % cli.image_ref) + if (cli.script is not None): + if (not ch.Path(cli.script).exists): + ch.FATAL("%s: no such file" % cli.script) + + # This kludge is necessary because cli is a global variable, with cli.tag + # assumed present elsewhere in the file. Here, cli.tag represents the + # destination image. + cli.tag = str(out_image) + + # We check that stdin isn’t None to ensure that we don’t go down this code + # path by mistake (e.g. in CI, where stdin will never by a TTY). + if ((not sys.stdin.isatty()) and (commands == [])): + stdin = sys.stdin.read() + if (stdin != ''): + # https://stackoverflow.com/a/6482200 + + # We use “decode("utf-8")” here because stdout seems default to a bytes + # object, which is not a valid type for an argument for “Path”. + tmpfile = ch.Path(subprocess.run(["mktemp", "-d"],capture_output=True).stdout.decode("utf-8")) + with open(tmpfile, "w") as outfile: + outfile.write(stdin) + # By default, the file is seemingly created with its execute bit + # unassigned. This is problematic for the RUN instruction. + os.chmod(tmpfile, 0o755) + cli.script = str(tmpfile) + + if (cli.shell is not None): + shell = cli.shell + else: + shell = "/bin/sh" + # FIXME: This logic is a bit busted + if ((commands != []) or (cli.script is not None)): + # non-interactive case (“-c” or script) + if (cli.script is not None): + tree = modify_tree_make_script(src_image.ref, cli.script) + else: + tree = modify_tree_make(src_image.ref, commands) + + # FIXME: pretty printing should prob go here, see issue #1908. + image_ct = sum(1 for i in tree.children_("from_")) + + # problem here from moving function out of build.py (build.py assumes cli + # is global variable) + ml = build.traverse_parse_tree(tree, image_ct, cli) + else: + # Interactive case + + # Generate “fake” SID for build cache. We do this because we can’t compute + # an SID, but we still want to make sure that it’s unique enough that + # we’re unlikely to run into a collision. + fake_sid = uuid.uuid4() + out_image.unpack_clear() + out_image.copy_unpacked(src_image) + bu.cache.worktree_adopt(out_image, src_image.ref.for_path) + bu.cache.ready(out_image) + bu.cache.branch_nocheckout(src_image.ref, out_image.ref) + foo = subprocess.run([ch.CH_BIN + "/ch-run", "--unsafe", "-w", + str(out_image.ref), "--", shell]) + if (foo.returncode == ch.Ch_Run_Retcode.ERR_CMD.value): + # FIXME: Write a better error message? + ch.FATAL("Unable to run shell: %s" % shell) + ch.VERBOSE("using SID %s" % fake_sid) + # FIXME: metadata history stuff? See misc.import_. + if (out_image.metadata["history"] == []): + out_image.metadata["history"].append({ "empty_layer": False, + "command": "ch-image import"}) + out_image.metadata_save() + bu.cache.commit(out_image.unpack_path, fake_sid, "MODIFY interactive", []) + +def modify_tree_make(src_img, cmds): + """Function that manually constructs a parse tree corresponding to a set of + “ch-image modify” commands, as though the commands had been specified in a + Dockerfile. Note that because “ch-image modify” simply executes one or + more commands inside a container, the only Dockerfile instructions we need + to consider are “FROM” and “RUN”. E.g. for the command line + + $ ch-image modify -c 'echo foo' -c 'echo bar' -- foo foo2 + + this function produces the following parse tree + + start + dockerfile + from_ + image_ref + IMAGE_REF foo + run + run_shell + LINE_CHUNK echo foo + run + run_shell + LINE_CHUNK echo bar + """ + # Children of dockerfile tree + df_children = [] + # Metadata attribute. We use this attribute in the “_pretty” method for our + # “Tree” class. Constructing a tree without specifying a “Meta” instance that + # has been given a “line” value will result in the attribute not being present, + # which causes an error when we try to access that attribute. Here we give the + # attribute a debug value of -1 to avoid said errors. + meta = lark.tree.Meta() + meta.line = -1 + df_children.append(im.Tree(lark.Token('RULE', 'from_'), + [im.Tree(lark.Token('RULE', 'image_ref'), + [lark.Token('IMAGE_REF', str(src_img))], meta)], meta)) + if (cli.shell is not None): + df_children.append(im.Tree(lark.Token('RULE', 'shell'), + [lark.Token('STRING_QUOTED', '"%s"' % cli.shell), + lark.Token('STRING_QUOTED', '"-c"')],meta)) + for cmd in cmds: + df_children.append(im.Tree(lark.Token('RULE', 'run'), + [im.Tree(lark.Token('RULE', 'run_shell'), + [lark.Token('LINE_CHUNK', cmd)], meta)],meta)) + return im.Tree(lark.Token('RULE', 'start'), [im.Tree(lark.Token('RULE','dockerfile'), df_children)], meta) + +# FIXME: Combine with “modify_tree_make”? +def modify_tree_make_script(src_img, path): + """Temporary(?) analog of “modify_tree_make” for the non-interactive version + of “modify” using a script. For the command line: + + $ ch-image modify foo foo2 /path/to/script + + this function produces the following parse tree + + start + dockerfile + from_ + image_ref + IMAGE_REF foo + copy + copy_shell + WORD /path/to/script WORD /ch/script.sh + run + run_shell + LINE_CHUNK /ch/script.sh + """ + # Children of dockerfile tree + df_children = [] + # Metadata attribute. We use this attribute in the “_pretty” method for our + # “Tree” class. Constructing a tree without specifying a “Meta” instance that + # has been given a “line” value will result in the attribute not being present, + # which causes an error when we try to access that attribute. Here we give the + # attribute a debug value of -1 to avoid said errors. + meta = lark.tree.Meta() + meta.line = -1 + df_children.append(im.Tree(lark.Token('RULE', 'from_'), + [im.Tree(lark.Token('RULE', 'image_ref'), + [lark.Token('IMAGE_REF', str(src_img))], meta)], meta)) + if (cli.shell is not None): + df_children.append(im.Tree(lark.Token('RULE', 'shell'), + [lark.Token('STRING_QUOTED', '"%s"' % cli.shell), + lark.Token('STRING_QUOTED', '"-c"')],meta)) + df_children.append(im.Tree(lark.Token('RULE', 'copy'), + [im.Tree(lark.Token('RULE', 'copy_shell'),[lark.Token('WORD', path), + lark.Token('WORD', '/ch/script.sh')], meta)],meta)) + df_children.append(im.Tree(lark.Token('RULE', 'run'), + [im.Tree(lark.Token('RULE', 'run_shell'), + [lark.Token('LINE_CHUNK', '/ch/script.sh')], meta)],meta)) + return im.Tree(lark.Token('RULE', 'start'), [im.Tree(lark.Token('RULE','dockerfile'), df_children)], meta) From d3b06fbdae828cf93a57e29c839e168d21c65f47 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Fri, 5 Jul 2024 16:01:49 +0000 Subject: [PATCH 52/65] add new module to makefile --- lib/Makefile.am | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/Makefile.am b/lib/Makefile.am index 8d8f1378c..0dbff3807 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -13,6 +13,7 @@ dist_mylib_DATA = base.sh \ force.py \ image.py \ misc.py \ + modify.py \ pull.py \ push.py \ registry.py From ce2a6200e835154869f5842ec089a1bfde2ea23e Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Fri, 5 Jul 2024 16:11:50 +0000 Subject: [PATCH 53/65] update fedora spec --- packaging/fedora/charliecloud.spec | 1 + 1 file changed, 1 insertion(+) diff --git a/packaging/fedora/charliecloud.spec b/packaging/fedora/charliecloud.spec index d2eac0262..ed5088da3 100644 --- a/packaging/fedora/charliecloud.spec +++ b/packaging/fedora/charliecloud.spec @@ -154,6 +154,7 @@ ln -s "${sphinxdir}/js" %{buildroot}%{_pkgdocdir}/html/_static/js %{_prefix}/lib/%{name}/lark %{_prefix}/lib/%{name}/lark-1.1.9.dist-info %{_prefix}/lib/%{name}/misc.py +%{_prefix}/lib/%{name}/modify.py %{_prefix}/lib/%{name}/pull.py %{_prefix}/lib/%{name}/push.py %{_prefix}/lib/%{name}/registry.py From 7f17900083b339dabdea6359e90d06ae0daae97e Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Fri, 5 Jul 2024 21:48:45 +0000 Subject: [PATCH 54/65] a few minor suggestions --- doc/ch-image.rst | 4 ++-- lib/modify.py | 2 +- test/build/50_ch-image.bats | 4 ++-- test/common.bash | 2 +- test/run/ch-run_misc.bats | 8 +------- 5 files changed, 7 insertions(+), 13 deletions(-) diff --git a/doc/ch-image.rst b/doc/ch-image.rst index 77278dc67..8264e83da 100644 --- a/doc/ch-image.rst +++ b/doc/ch-image.rst @@ -2106,7 +2106,7 @@ in the remote registry, so we don’t upload it again.) Delete all images and cache from ch-image builder storage. -:code:`shell` +:code:`modify` ============= Modify an image with shell commands, possibly interactively. @@ -2164,7 +2164,7 @@ That is, :code:`ch-image` simply builds a Dockerfile internally that uses executes this Dockerfile to produce image :code:`bar`. That is, if any command fails, the build fails and no further commands are attempted. -This mode provides a detailed image provenance just like a Dockerfile. +This mode provides detailed image provenance just like a Dockerfile. Non-interactive mode using a shell script ----------------------------------------- diff --git a/lib/modify.py b/lib/modify.py index 9f985b661..99e6d9e43 100644 --- a/lib/modify.py +++ b/lib/modify.py @@ -104,7 +104,7 @@ def main(cli_): str(out_image.ref), "--", shell]) if (foo.returncode == ch.Ch_Run_Retcode.ERR_CMD.value): # FIXME: Write a better error message? - ch.FATAL("Unable to run shell: %s" % shell) + ch.FATAL("can't run shell: %s" % shell) ch.VERBOSE("using SID %s" % fake_sid) # FIXME: metadata history stuff? See misc.import_. if (out_image.metadata["history"] == []): diff --git a/test/build/50_ch-image.bats b/test/build/50_ch-image.bats index 389802ab9..d7ca05bd2 100644 --- a/test/build/50_ch-image.bats +++ b/test/build/50_ch-image.bats @@ -1020,9 +1020,9 @@ EOF [[ $status -eq 1 ]] [[ $output = *'output must be different from source image'* ]] - # non-existant shell + # non-existent shell run ch-image modify -S "doesnotexist" -- alpine:3.17 tmpimg echo "$output" [[ $status -eq 1 ]] - [[ $output = *'Unable to run shell:'* ]] + [[ $output = *"can't run shell:"* ]] } diff --git a/test/common.bash b/test/common.bash index 373833563..beb948051 100644 --- a/test/common.bash +++ b/test/common.bash @@ -332,7 +332,7 @@ export BATS_TMPDIR=$btnew # ch-run exit codes. (see also: ch_misc.h, lib/build.py) CH_ERR_RUN=31 CH_ERR_CMD=49 -CH_ERR_SQUASH=84 # Currently not used, here just in case +#CH_ERR_SQUASH=84 # Currently not used ch_runfile=$(command -v ch-run) diff --git a/test/run/ch-run_misc.bats b/test/run/ch-run_misc.bats index 53a6129e9..e5a325788 100644 --- a/test/run/ch-run_misc.bats +++ b/test/run/ch-run_misc.bats @@ -1,4 +1,4 @@ - load ../common +load ../common bind1_dir=$BATS_TMPDIR/bind1 bind2_dir=$BATS_TMPDIR/bind2 @@ -322,18 +322,12 @@ EOF # empty argument to --bind run ch-run -b '' "$ch_timg" -- /bin/true echo "$output" - echo "STATUS" - echo "$status" - echo "STATUS" [[ $status -eq $CH_ERR_RUN ]] [[ $output = *'--bind: no source provided'* ]] # source not provided run ch-run -b :/mnt/9 "$ch_timg" -- /bin/true echo "$output" - echo "STATUS" - echo "$status" - echo "STATUS" [[ $status -eq $CH_ERR_RUN ]] [[ $output = *'--bind: no source provided'* ]] From 8d0befcafe2792ce492c87d40c63b23f7aa828d1 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Mon, 15 Jul 2024 17:39:57 +0000 Subject: [PATCH 55/65] more suggestions, move some build cli stuff around --- bin/ch-checkns.c | 2 +- bin/ch-image.py.in | 16 ++++- bin/ch-run.c | 6 +- bin/ch_core.c | 2 +- bin/ch_fuse.c | 2 +- bin/ch_misc.c | 2 +- bin/ch_misc.h | 6 +- doc/ch-image.rst | 2 +- lib/build.py | 64 +++++++++---------- lib/charliecloud.py | 6 +- lib/modify.py | 147 ++++++++++++++++++++++++++++---------------- lib/pull.py | 2 +- 12 files changed, 156 insertions(+), 101 deletions(-) diff --git a/bin/ch-checkns.c b/bin/ch-checkns.c index 6487ebe1f..c4d02e8da 100644 --- a/bin/ch-checkns.c +++ b/bin/ch-checkns.c @@ -71,7 +71,7 @@ void fatal_(const char *file, int line, int errno_, const char *str) char *url = "https://github.com/hpc/charliecloud/blob/master/bin/ch-checkns.c"; printf("error: %s: %d: %s\n", file, line, str); printf("errno: %d\nsee: %s\n", errno_, url); - exit(ERR_CHRUN); + exit(EXIT_CHRUN); } int main(int argc, char *argv[]) diff --git a/bin/ch-image.py.in b/bin/ch-image.py.in index 61802cbc2..a7bcc9a44 100644 --- a/bin/ch-image.py.in +++ b/bin/ch-image.py.in @@ -284,11 +284,25 @@ def main(): sp = ap.add_parser("modify", "foo") add_opts(sp, modify.main, deps_check=True, stog_init=True) sp.add_argument("-c", metavar="CMD", action="append", default=[], nargs=1, help="foo") - sp.add_argument("-S", "--shell", metavar="shell", help="foo") + sp.add_argument("-i", "--interactive", action="store_true", help="foo") + sp.add_argument("-t", "--test", action="store_true", help="foo") + sp.add_argument("-S", "--shell", metavar="shell", default="/bin/sh", help="foo") sp.add_argument("image_ref", metavar="IMAGE_REF", help="image to modify") sp.add_argument("out_image", metavar="OUT_IMAGE", help="destination of modified image") # Optional positional argument? https://stackoverflow.com/a/4480202 sp.add_argument("script", metavar="SCRIPT", help="foo", nargs='?') + sp.add_argument("-b", "--bind", metavar="SRC[:DST]", + action="append", default=[], + help="mount SRC at guest DST (default: same as SRC)") + sp.add_argument("--build-arg", metavar="ARG[=VAL]", + action="append", default=[], + help="set build-time variable ARG to VAL, or $ARG if no VAL") + sp.add_argument("--force", metavar="MODE", nargs="?", default="seccomp", + type=ch.Force_Mode, const="seccomp", + help="inject unprivileged build workarounds") + sp.add_argument("--force-cmd", metavar="CMD,ARG1[,ARG2...]", + action="append", default=[], + help="command arg(s) to add under --force=seccomp") # pull sp = ap.add_parser("pull", diff --git a/bin/ch-run.c b/bin/ch-run.c index 16111824b..89630e9b7 100644 --- a/bin/ch-run.c +++ b/bin/ch-run.c @@ -445,7 +445,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) #ifdef HAVE_FNM_EXTMATCH exit(0); #else - exit(ERR_CHRUN); + exit(EXIT_CHRUN); #endif } else if (!strcmp(arg, "overlayfs")) { #ifdef HAVE_OVERLAYFS @@ -457,13 +457,13 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) #ifdef HAVE_SECCOMP exit(0); #else - exit(ERR_CHRUN); + exit(EXIT_CHRUN); #endif } else if (!strcmp(arg, "squash")) { #ifdef HAVE_LIBSQUASHFUSE exit(0); #else - exit(ERR_CHRUN); + exit(EXIT_CHRUN); #endif } else if (!strcmp(arg, "tmpfs-xattrs")) { #ifdef HAVE_TMPFS_XATTRS diff --git a/bin/ch_core.c b/bin/ch_core.c index 92b5b72a3..60dcd1097 100644 --- a/bin/ch_core.c +++ b/bin/ch_core.c @@ -568,7 +568,7 @@ void run_user_command(char *argv[], const char *initial_dir) T_ (freopen("/dev/null", "w", stderr)); execvp(argv[0], argv); // only returns if error ERROR(errno, "can't execve(2): %s", argv[0]); - exit(ERR_CMD); + exit(EXIT_CMD); } /* Set up the fake-syscall seccomp(2) filter. This computes and installs a diff --git a/bin/ch_fuse.c b/bin/ch_fuse.c index e8cbcbc55..22ac1e5ca 100644 --- a/bin/ch_fuse.c +++ b/bin/ch_fuse.c @@ -188,7 +188,7 @@ int sq_loop(void) // Clean up zombie child if exit signal was SIGCHLD. if (!sigchld_received) - exit_code = ERR_SQUASH; + exit_code = EXIT_SQUASH; else { Tf (wait(&child_status) >= 0, "can't wait for child"); if (WIFEXITED(child_status)) { diff --git a/bin/ch_misc.c b/bin/ch_misc.c index 674fdb2df..8de09e3c1 100644 --- a/bin/ch_misc.c +++ b/bin/ch_misc.c @@ -605,7 +605,7 @@ noreturn void msg_fatal(const char *file, int line, int errno_, msgv(LL_FATAL, file, line, errno_, fmt, ap); va_end(ap); - exit(ERR_CHRUN); + exit(EXIT_CHRUN); } /* va_list form of msg(). */ diff --git a/bin/ch_misc.h b/bin/ch_misc.h index 8eee99d1f..5768d54c4 100644 --- a/bin/ch_misc.h +++ b/bin/ch_misc.h @@ -25,9 +25,9 @@ #define WARNINGS_SIZE (4*1024) /* Exit codes (see also: test/common.bash, lib/build.py). */ -#define ERR_CHRUN 31 -#define ERR_CMD 49 -#define ERR_SQUASH 84 +#define EXIT_CHRUN 31 +#define EXIT_CMD 49 +#define EXIT_SQUASH 84 /* Test some value, and if it's not what we expect, exit with a fatal error. These are macros so we have access to the file and line number. diff --git a/doc/ch-image.rst b/doc/ch-image.rst index 8264e83da..518b3a55a 100644 --- a/doc/ch-image.rst +++ b/doc/ch-image.rst @@ -2107,7 +2107,7 @@ Delete all images and cache from ch-image builder storage. :code:`modify` -============= +============== Modify an image with shell commands, possibly interactively. diff --git a/lib/build.py b/lib/build.py index 28eae8afd..e5f0c3e8b 100644 --- a/lib/build.py +++ b/lib/build.py @@ -134,7 +134,7 @@ def main(cli_): tree = parse_dockerfile(text) global image_ct - ml = traverse_parse_tree(tree, image_ct, cli) + ml = parse_tree_traverse(tree, image_ct, cli) ## Functions ## @@ -144,35 +144,6 @@ def main(cli_): # reference, so changes made to mutable objects (which “cli” is) will persist in # the scope of the caller.' def cli_process(cli): - # Infer input file if needed. - if (cli.file is None): - cli.file = cli.context + "/Dockerfile" - - # Infer image name if needed. - if (cli.tag is None): - path = os.path.basename(cli.file) - if ("." in path): - (base, ext_all) = str(path).split(".", maxsplit=1) - (base_all, ext_last) = str(path).rsplit(".", maxsplit=1) - else: - base = None - ext_last = None - if (base == "Dockerfile"): - cli.tag = ext_all - ch.VERBOSE("inferring name from Dockerfile extension: %s" % cli.tag) - elif (ext_last in ("df", "dockerfile")): - cli.tag = base_all - ch.VERBOSE("inferring name from Dockerfile basename: %s" % cli.tag) - elif (os.path.abspath(cli.context) != "/"): - cli.tag = os.path.basename(os.path.abspath(cli.context)) - ch.VERBOSE("inferring name from context directory: %s" % cli.tag) - else: - assert (os.path.abspath(cli.context) == "/") - cli.tag = "root" - ch.VERBOSE("inferring name with root context directory: %s" % cli.tag) - cli.tag = re.sub(r"[^a-z0-9_.-]", "", cli.tag.lower()) - ch.INFO("inferred image name: %s" % cli.tag) - # --force and friends. if (cli.force_cmd and cli.force == ch.Force_Mode.FAKEROOT): ch.FATAL("--force-cmd and --force=fakeroot are incompatible") @@ -206,6 +177,37 @@ def build_arg_get(arg): ch.FATAL("--build-arg: %s: no value and not in environment" % kv[0]) return (kv[0], v) cli.build_arg = dict( build_arg_get(i) for i in cli.build_arg ) + + # Infer input file if needed. + if (cli.file is None): + cli.file = cli.context + "/Dockerfile" + + # Infer image name if needed. + if (cli.tag is None): + path = os.path.basename(cli.file) + if ("." in path): + (base, ext_all) = str(path).split(".", maxsplit=1) + (base_all, ext_last) = str(path).rsplit(".", maxsplit=1) + else: + base = None + ext_last = None + if (base == "Dockerfile"): + cli.tag = ext_all + ch.VERBOSE("inferring name from Dockerfile extension: %s" % cli.tag) + elif (ext_last in ("df", "dockerfile")): + cli.tag = base_all + ch.VERBOSE("inferring name from Dockerfile basename: %s" % cli.tag) + elif (os.path.abspath(cli.context) != "/"): + cli.tag = os.path.basename(os.path.abspath(cli.context)) + ch.VERBOSE("inferring name from context directory: %s" % cli.tag) + else: + assert (os.path.abspath(cli.context) == "/") + cli.tag = "root" + ch.VERBOSE("inferring name with root context directory: %s" % cli.tag) + cli.tag = re.sub(r"[^a-z0-9_.-]", "", cli.tag.lower()) + ch.INFO("inferred image name: %s" % cli.tag) + + ch.DEBUG(cli) # Guess whether the context is a URL, and error out if so. This can be a @@ -280,7 +282,7 @@ def parse_dockerfile(text): # [1]: https://lark-parser.readthedocs.io/en/latest/visitors/#visitors # [2]: https://github.com/lark-parser/lark/blob/445c8d4/lark/visitors.py#L211 # [3]: https://lark-parser.readthedocs.io/en/latest/classes/#tree -def traverse_parse_tree(tree, image_ct_, cli_): +def parse_tree_traverse(tree, image_ct_, cli_): global cli global image_ct cli = cli_ diff --git a/lib/charliecloud.py b/lib/charliecloud.py index 29de18ca8..3f729ed0a 100644 --- a/lib/charliecloud.py +++ b/lib/charliecloud.py @@ -43,9 +43,9 @@ class Build_Mode(enum.Enum): # ch-run exit codes (see also: bin/ch_misc.h) class Ch_Run_Retcode(enum.Enum): - ERR_CHRUN = 31 - ERR_CMD = 49 - ERR_SQUASH = 84 + EXIT_CHRUN = 31 + EXIT_CMD = 49 + EXIT_SQUASH = 84 # Download cache mode. class Download_Mode(enum.Enum): diff --git a/lib/modify.py b/lib/modify.py index 99e6d9e43..7d16f6825 100644 --- a/lib/modify.py +++ b/lib/modify.py @@ -1,5 +1,6 @@ # implementation of ch-image modify +import enum import os import subprocess import sys @@ -13,6 +14,11 @@ lark = im.lark +class Modify_Mode(enum.Enum): + COMMAND_SEQ = "commands" + INTERACTIVE = "interactive" + SCRIPT = "script" + def main(cli_): global called called = True @@ -28,10 +34,10 @@ def main(cli_): # don’t assign these values, the program will fail after trying to access # them. FIXME: Can partially fix this by adding command line opts. cli.parse_only = False - cli.force = ch.Force_Mode.SECCOMP + #cli.force = ch.Force_Mode.SECCOMP cli.force_cmd = force.FORCE_CMD_DEFAULT - cli.bind = [] - cli.build_arg = [] + #cli.bind = [] + #cli.build_arg = [] cli.context = os.path.abspath(os.sep) commands = [] @@ -53,42 +59,46 @@ def main(cli_): # destination image. cli.tag = str(out_image) - # We check that stdin isn’t None to ensure that we don’t go down this code - # path by mistake (e.g. in CI, where stdin will never by a TTY). - if ((not sys.stdin.isatty()) and (commands == [])): - stdin = sys.stdin.read() - if (stdin != ''): - # https://stackoverflow.com/a/6482200 - - # We use “decode("utf-8")” here because stdout seems default to a bytes - # object, which is not a valid type for an argument for “Path”. - tmpfile = ch.Path(subprocess.run(["mktemp", "-d"],capture_output=True).stdout.decode("utf-8")) - with open(tmpfile, "w") as outfile: - outfile.write(stdin) - # By default, the file is seemingly created with its execute bit - # unassigned. This is problematic for the RUN instruction. - os.chmod(tmpfile, 0o755) - cli.script = str(tmpfile) - - if (cli.shell is not None): - shell = cli.shell - else: - shell = "/bin/sh" - # FIXME: This logic is a bit busted - if ((commands != []) or (cli.script is not None)): - # non-interactive case (“-c” or script) + # Determine modify mode based on what is present in command line + if (commands != []): + if (cli.interactive): + ch.FATAL(">:(") if (cli.script is not None): - tree = modify_tree_make_script(src_image.ref, cli.script) - else: - tree = modify_tree_make(src_image.ref, commands) - - # FIXME: pretty printing should prob go here, see issue #1908. - image_ct = sum(1 for i in tree.children_("from_")) - - # problem here from moving function out of build.py (build.py assumes cli - # is global variable) - ml = build.traverse_parse_tree(tree, image_ct, cli) + ch.FATAL(">:(") + mode = Modify_Mode.COMMAND_SEQ + elif (cli.script is not None): + if (cli.interactive): + ch.FATAL(">:(") + mode = Modify_Mode.SCRIPT + elif (sys.stdin.isatty() or (cli.interactive)): + mode = Modify_Mode.INTERACTIVE else: + # copy stdin to tmp file + # if tmp file is empty error out + # goto filename argument present, mode N + stdin = sys.stdin.read() + if (stdin == ''): + ch.FATAL("modify mode unclear") + + # We use “decode("utf-8")” here because stdout seems default to a bytes + # object, which is not a valid type for an argument for “Path”. + tmpfile = ch.Path(subprocess.run(["mktemp"],capture_output=True).stdout.decode("utf-8")) + # https://stackoverflow.com/a/6482200 + with open(tmpfile, "w") as outfile: + outfile.write(stdin) + # By default, the file is seemingly created with its execute bit + # unassigned. This is problematic for the RUN instruction. + os.chmod(tmpfile, 0o755) + cli.script = str(tmpfile) + mode = Modify_Mode.SCRIPT + + ch.VERBOSE("shell: %s" % cli.shell) + + ch.ILLERI("mode: %s" % mode) + if (cli.test): + exit(0) + + if (mode == Modify_Mode.INTERACTIVE): # Interactive case # Generate “fake” SID for build cache. We do this because we can’t compute @@ -100,11 +110,12 @@ def main(cli_): bu.cache.worktree_adopt(out_image, src_image.ref.for_path) bu.cache.ready(out_image) bu.cache.branch_nocheckout(src_image.ref, out_image.ref) - foo = subprocess.run([ch.CH_BIN + "/ch-run", "--unsafe", "-w", - str(out_image.ref), "--", shell]) - if (foo.returncode == ch.Ch_Run_Retcode.ERR_CMD.value): + foo = subprocess.run([ch.CH_BIN + "/ch-run", "--unsafe", "-w"] + + sum([["-b", i] for i in cli.bind], []) + + [str(out_image.ref), "--", cli.shell]) + if (foo.returncode == ch.Ch_Run_Retcode.EXIT_CMD.value): # FIXME: Write a better error message? - ch.FATAL("can't run shell: %s" % shell) + ch.FATAL("can't run shell: %s" % cli.shell) ch.VERBOSE("using SID %s" % fake_sid) # FIXME: metadata history stuff? See misc.import_. if (out_image.metadata["history"] == []): @@ -112,13 +123,28 @@ def main(cli_): "command": "ch-image import"}) out_image.metadata_save() bu.cache.commit(out_image.unpack_path, fake_sid, "MODIFY interactive", []) + else: + # non-interactive case + if (mode == Modify_Mode.SCRIPT): + # script specified + tree = modify_tree_make_script(src_image.ref, cli.script) + elif (mode == Modify_Mode.COMMAND_SEQ): + # “-c” specified + tree = modify_tree_make(src_image.ref, commands) + else: + assert False, "unreachable code reached" + + # FIXME: pretty printing should prob go here, see issue #1908. + image_ct = sum(1 for i in tree.children_("from_")) + + build.parse_tree_traverse(tree, image_ct, cli) def modify_tree_make(src_img, cmds): - """Function that manually constructs a parse tree corresponding to a set of - “ch-image modify” commands, as though the commands had been specified in a - Dockerfile. Note that because “ch-image modify” simply executes one or - more commands inside a container, the only Dockerfile instructions we need - to consider are “FROM” and “RUN”. E.g. for the command line + """Construct a parse tree corresponding to a set of “ch-image modify” + commands, as though the commands had been specified in a Dockerfile. Note + that because “ch-image modify” simply executes one or more commands inside + a container, the only Dockerfile instructions we need to consider are + “FROM” and “RUN”. E.g. for the command line $ ch-image modify -c 'echo foo' -c 'echo bar' -- foo foo2 @@ -147,15 +173,20 @@ def modify_tree_make(src_img, cmds): meta.line = -1 df_children.append(im.Tree(lark.Token('RULE', 'from_'), [im.Tree(lark.Token('RULE', 'image_ref'), - [lark.Token('IMAGE_REF', str(src_img))], meta)], meta)) + [lark.Token('IMAGE_REF', str(src_img))], + meta) + ], meta)) if (cli.shell is not None): df_children.append(im.Tree(lark.Token('RULE', 'shell'), [lark.Token('STRING_QUOTED', '"%s"' % cli.shell), - lark.Token('STRING_QUOTED', '"-c"')],meta)) + lark.Token('STRING_QUOTED', '"-c"') + ],meta)) for cmd in cmds: df_children.append(im.Tree(lark.Token('RULE', 'run'), [im.Tree(lark.Token('RULE', 'run_shell'), - [lark.Token('LINE_CHUNK', cmd)], meta)],meta)) + [lark.Token('LINE_CHUNK', cmd)], + meta) + ], meta)) return im.Tree(lark.Token('RULE', 'start'), [im.Tree(lark.Token('RULE','dockerfile'), df_children)], meta) # FIXME: Combine with “modify_tree_make”? @@ -190,15 +221,23 @@ def modify_tree_make_script(src_img, path): meta.line = -1 df_children.append(im.Tree(lark.Token('RULE', 'from_'), [im.Tree(lark.Token('RULE', 'image_ref'), - [lark.Token('IMAGE_REF', str(src_img))], meta)], meta)) + [lark.Token('IMAGE_REF', str(src_img))], + meta) + ], meta)) if (cli.shell is not None): df_children.append(im.Tree(lark.Token('RULE', 'shell'), [lark.Token('STRING_QUOTED', '"%s"' % cli.shell), - lark.Token('STRING_QUOTED', '"-c"')],meta)) + lark.Token('STRING_QUOTED', '"-c"') + ],meta)) df_children.append(im.Tree(lark.Token('RULE', 'copy'), - [im.Tree(lark.Token('RULE', 'copy_shell'),[lark.Token('WORD', path), - lark.Token('WORD', '/ch/script.sh')], meta)],meta)) + [im.Tree(lark.Token('RULE', 'copy_shell'), + [lark.Token('WORD', path), + lark.Token('WORD', '/ch/script.sh') + ], meta) + ],meta)) df_children.append(im.Tree(lark.Token('RULE', 'run'), [im.Tree(lark.Token('RULE', 'run_shell'), - [lark.Token('LINE_CHUNK', '/ch/script.sh')], meta)],meta)) + [lark.Token('LINE_CHUNK', '/ch/script.sh')], + meta) + ], meta)) return im.Tree(lark.Token('RULE', 'start'), [im.Tree(lark.Token('RULE','dockerfile'), df_children)], meta) diff --git a/lib/pull.py b/lib/pull.py index cea1ec842..9e52b7d85 100644 --- a/lib/pull.py +++ b/lib/pull.py @@ -106,7 +106,7 @@ def download(self): # currently, this error is only raised if we’ve downloaded the # skinny manifest. have_skinny = True - if (ch.arch in ["amd64", "aarch64"]): + if (ch.arch in "amd64"): # We’re guessing that enough arch-unaware images are amd64 to # barge ahead if requested architecture is amd64. ch.arch = "yolo" From 1b65df937867d891c690e24b762f17e535d4536b Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Mon, 15 Jul 2024 18:12:36 +0000 Subject: [PATCH 56/65] fix CI tty error --- test/build/50_ch-image.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/build/50_ch-image.bats b/test/build/50_ch-image.bats index d7ca05bd2..33c5188b3 100644 --- a/test/build/50_ch-image.bats +++ b/test/build/50_ch-image.bats @@ -1021,7 +1021,7 @@ EOF [[ $output = *'output must be different from source image'* ]] # non-existent shell - run ch-image modify -S "doesnotexist" -- alpine:3.17 tmpimg + run ch-image modify -i -S "doesnotexist" -- alpine:3.17 tmpimg echo "$output" [[ $status -eq 1 ]] [[ $output = *"can't run shell:"* ]] From d4f90f3b84414de02b2e4722f7d932af1d388130 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Mon, 15 Jul 2024 20:25:27 +0000 Subject: [PATCH 57/65] add cache validation test --- lib/build.py | 72 +++++++++++++++++++++------------------- lib/modify.py | 9 +++-- test/build/55_cache.bats | 42 +++++++++++++++++++++++ 3 files changed, 84 insertions(+), 39 deletions(-) diff --git a/lib/build.py b/lib/build.py index e5f0c3e8b..de1819ec0 100644 --- a/lib/build.py +++ b/lib/build.py @@ -127,6 +127,8 @@ def main(cli_): global cli cli = cli_ + cli_process_common(cli) + # Process CLI. Make appropriate modifications to “cli” instance and return # Dockerfile text. text = cli_process(cli) @@ -144,40 +146,6 @@ def main(cli_): # reference, so changes made to mutable objects (which “cli” is) will persist in # the scope of the caller.' def cli_process(cli): - # --force and friends. - if (cli.force_cmd and cli.force == ch.Force_Mode.FAKEROOT): - ch.FATAL("--force-cmd and --force=fakeroot are incompatible") - if (not cli.force_cmd): - cli.force_cmd = force.FORCE_CMD_DEFAULT - else: - cli.force = ch.Force_Mode.SECCOMP - # convert cli.force_cmd to parsed dict - force_cmd = dict() - for line in cli.force_cmd: - (cmd, args) = force.force_cmd_parse(line) - force_cmd[cmd] = args - cli.force_cmd = force_cmd - ch.VERBOSE("force mode: %s" % cli.force) - if (cli.force == ch.Force_Mode.SECCOMP): - for (cmd, args) in cli.force_cmd.items(): - ch.VERBOSE("force command: %s" % ch.argv_to_string([cmd] + args)) - if ( cli.force == ch.Force_Mode.SECCOMP - and ch.cmd([ch.CH_BIN + "/ch-run", "--feature=seccomp"], - fail_ok=True) != 0): - ch.FATAL("ch-run was not built with seccomp(2) support") - - # Deal with build arguments. - def build_arg_get(arg): - kv = arg.split("=") - if (len(kv) == 2): - return kv - else: - v = os.getenv(kv[0]) - if (v is None): - ch.FATAL("--build-arg: %s: no value and not in environment" % kv[0]) - return (kv[0], v) - cli.build_arg = dict( build_arg_get(i) for i in cli.build_arg ) - # Infer input file if needed. if (cli.file is None): cli.file = cli.context + "/Dockerfile" @@ -234,6 +202,42 @@ def build_arg_get(arg): return text +# Process common opts between modify and build. +def cli_process_common(cli): + # --force and friends. + if (cli.force_cmd and cli.force == ch.Force_Mode.FAKEROOT): + ch.FATAL("--force-cmd and --force=fakeroot are incompatible") + if (not cli.force_cmd): + cli.force_cmd = force.FORCE_CMD_DEFAULT + else: + cli.force = ch.Force_Mode.SECCOMP + # convert cli.force_cmd to parsed dict + force_cmd = dict() + for line in cli.force_cmd: + (cmd, args) = force.force_cmd_parse(line) + force_cmd[cmd] = args + cli.force_cmd = force_cmd + ch.VERBOSE("force mode: %s" % cli.force) + if (cli.force == ch.Force_Mode.SECCOMP): + for (cmd, args) in cli.force_cmd.items(): + ch.VERBOSE("force command: %s" % ch.argv_to_string([cmd] + args)) + if ( cli.force == ch.Force_Mode.SECCOMP + and ch.cmd([ch.CH_BIN + "/ch-run", "--feature=seccomp"], + fail_ok=True) != 0): + ch.FATAL("ch-run was not built with seccomp(2) support") + + # Deal with build arguments. + def build_arg_get(arg): + kv = arg.split("=") + if (len(kv) == 2): + return kv + else: + v = os.getenv(kv[0]) + if (v is None): + ch.FATAL("--build-arg: %s: no value and not in environment" % kv[0]) + return (kv[0], v) + cli.build_arg = dict( build_arg_get(i) for i in cli.build_arg ) + def parse_dockerfile(text): # Parse it. parser = lark.Lark(im.GRAMMAR_DOCKERFILE, parser="earley", diff --git a/lib/modify.py b/lib/modify.py index 7d16f6825..443ad7af9 100644 --- a/lib/modify.py +++ b/lib/modify.py @@ -30,16 +30,15 @@ def main(cli_): global cli cli = cli_ - # This file assumes that global cli comes from the “build” function. If we + # build.py assumes that global cli comes from the “build” function. If we # don’t assign these values, the program will fail after trying to access # them. FIXME: Can partially fix this by adding command line opts. cli.parse_only = False - #cli.force = ch.Force_Mode.SECCOMP - cli.force_cmd = force.FORCE_CMD_DEFAULT - #cli.bind = [] - #cli.build_arg = [] + #cli.force_cmd = force.FORCE_CMD_DEFAULT cli.context = os.path.abspath(os.sep) + build.cli_process_common(cli) + commands = [] # “Flatten” commands array for c in cli.c: diff --git a/test/build/55_cache.bats b/test/build/55_cache.bats index 2c125484a..5fd480eaa 100644 --- a/test/build/55_cache.bats +++ b/test/build/55_cache.bats @@ -1580,3 +1580,45 @@ EOF [[ $status -eq 0 ]] [[ $output != *'image erroneously marked cached, fixing'* ]] } + +@test "${tag}: modify" { + ch-image build-cache --reset + + ch-image pull alpine:3.17 + ch-image modify -c "echo foo" -c "echo bar" -- alpine:3.17 tmpimg + + blessed_out=$(cat << 'EOF' +* (tmpimg) RUN.S echo bar +* RUN.S echo foo +* SHELL ['/bin/sh', '-c'] +* (alpine+3.17) PULL alpine:3.17 +* (root) ROOT +EOF +) + + run ch-image build-cache --tree + echo "$output" + [[ $status -eq 0 ]] + diff -u <(echo "$blessed_out") <(echo "$output" | treeonly) + + echo "touch /home/bar" >> "${BATS_TMPDIR}/script.sh" + chmod 755 "${BATS_TMPDIR}/script.sh" + ch-image modify alpine:3.17 tmpimg "${BATS_TMPDIR}/script.sh" + + blessed_out=$(cat < '/ch/script.sh' +| * RUN.S echo bar +| * RUN.S echo foo +|/ +* SHELL ['/bin/sh', '-c'] +* (alpine+3.17) PULL alpine:3.17 +* (root) ROOT +EOF +) + + run ch-image build-cache --tree + echo "$output" + [[ $status -eq 0 ]] + diff -u <(echo "$blessed_out") <(echo "$output" | treeonly) +} From 09f78f2bbbb8d24d8c51a2f0953220fb2240789f Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Tue, 16 Jul 2024 15:54:42 +0000 Subject: [PATCH 58/65] add cache validation for interactive modify --- doc/ch-image.rst | 10 +++++----- lib/build.py | 3 ++- lib/modify.py | 1 - test/build/55_cache.bats | 21 +++++++++++++++++++++ 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/doc/ch-image.rst b/doc/ch-image.rst index 518b3a55a..e2bbadfd1 100644 --- a/doc/ch-image.rst +++ b/doc/ch-image.rst @@ -2158,11 +2158,11 @@ and:: RUN echo world EOF -That is, :code:`ch-image` simply builds a Dockerfile internally that uses -:code:`foo` as a base image, starts with an appropriate :code:`SHELL` if -:code:`-S` was given, converts each :code:`-c` to a :code:`RUN` command, and -executes this Dockerfile to produce image :code:`bar`. That is, if any command -fails, the build fails and no further commands are attempted. +:code:`ch-image` simply builds a Dockerfile internally that uses :code:`foo` as +a base image, starts with an appropriate :code:`SHELL` if :code:`-S` was given, +converts each :code:`-c` to a :code:`RUN` command, and executes this Dockerfile +to produce image :code:`bar`. As with regular Dockerfiles, if any command fails, +the build fails and no further commands are attempted. This mode provides detailed image provenance just like a Dockerfile. diff --git a/lib/build.py b/lib/build.py index de1819ec0..59998856d 100644 --- a/lib/build.py +++ b/lib/build.py @@ -61,7 +61,8 @@ class Environment: # Class responsible for traversing the parse tree generated by lark. “Main_Loop” # visits each node in the parse tree and calls its “__default__” method to # figure out what to do with the node. This behavior is defined by the parent -# class, “lark.Visitor”, documented here: https://lark-parser.readthedocs.io/en/latest/visitors.html +# class, “lark.Visitor”, documented here: +# https://lark-parser.readthedocs.io/en/latest/visitors.html class Main_Loop(lark.Visitor): __slots__ = ("instruction_total_ct", diff --git a/lib/modify.py b/lib/modify.py index 443ad7af9..3351fc74d 100644 --- a/lib/modify.py +++ b/lib/modify.py @@ -34,7 +34,6 @@ def main(cli_): # don’t assign these values, the program will fail after trying to access # them. FIXME: Can partially fix this by adding command line opts. cli.parse_only = False - #cli.force_cmd = force.FORCE_CMD_DEFAULT cli.context = os.path.abspath(os.sep) build.cli_process_common(cli) diff --git a/test/build/55_cache.bats b/test/build/55_cache.bats index 5fd480eaa..22d8e07d3 100644 --- a/test/build/55_cache.bats +++ b/test/build/55_cache.bats @@ -1615,6 +1615,27 @@ EOF * (alpine+3.17) PULL alpine:3.17 * (root) ROOT EOF +) + + run ch-image build-cache --tree + echo "$output" + [[ $status -eq 0 ]] + diff -u <(echo "$blessed_out") <(echo "$output" | treeonly) + + printf 'echo hello\nexit\n' | ch-image modify -i alpine:3.17 tmpimg + + blessed_out=$(cat < '/ch/script.sh' +| | * RUN.S echo bar +| | * RUN.S echo foo +| |/ +| * SHELL ['/bin/sh', '-c'] +|/ +* (alpine+3.17) PULL alpine:3.17 +* (root) ROOT +EOF ) run ch-image build-cache --tree From 681a39cd14ba9f9a5d936f4131a62a3c47fe76e6 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Thu, 18 Jul 2024 00:05:27 +0000 Subject: [PATCH 59/65] more suggestions --- lib/build.py | 9 ++++----- lib/modify.py | 39 ++++++++++++++++----------------------- 2 files changed, 20 insertions(+), 28 deletions(-) diff --git a/lib/build.py b/lib/build.py index 59998856d..091ef1c24 100644 --- a/lib/build.py +++ b/lib/build.py @@ -136,8 +136,11 @@ def main(cli_): tree = parse_dockerfile(text) + # Count the number of stages (i.e., FROM instructions) global image_ct - ml = parse_tree_traverse(tree, image_ct, cli) + image_ct = sum(1 for i in tree.children_("from_")) + + parse_tree_traverse(tree, image_ct, cli) ## Functions ## @@ -258,10 +261,6 @@ def parse_dockerfile(text): if (cli.parse_only): ch.exit(0) - # Count the number of stages (i.e., FROM instructions) - global image_ct - image_ct = sum(1 for i in tree.children_("from_")) - # If we use RSYNC, error out quickly if appropriate rsync(1) not present. if (tree.child("rsync") is not None): try: diff --git a/lib/modify.py b/lib/modify.py index 3351fc74d..219241007 100644 --- a/lib/modify.py +++ b/lib/modify.py @@ -4,6 +4,7 @@ import os import subprocess import sys +import tempfile import uuid import charliecloud as ch @@ -60,37 +61,33 @@ def main(cli_): # Determine modify mode based on what is present in command line if (commands != []): if (cli.interactive): - ch.FATAL(">:(") + ch.FATAL("incompatible opts: “-c”, “-i”") if (cli.script is not None): - ch.FATAL(">:(") + ch.FATAL("script mode incompatible with command mode") mode = Modify_Mode.COMMAND_SEQ elif (cli.script is not None): if (cli.interactive): - ch.FATAL(">:(") + ch.FATAL("script mode incompatible with interactive mode") mode = Modify_Mode.SCRIPT elif (sys.stdin.isatty() or (cli.interactive)): mode = Modify_Mode.INTERACTIVE else: - # copy stdin to tmp file - # if tmp file is empty error out - # goto filename argument present, mode N + # Write stdin to tempfile, copy tempfile into container as a script, run + # script. stdin = sys.stdin.read() if (stdin == ''): ch.FATAL("modify mode unclear") - # We use “decode("utf-8")” here because stdout seems default to a bytes - # object, which is not a valid type for an argument for “Path”. - tmpfile = ch.Path(subprocess.run(["mktemp"],capture_output=True).stdout.decode("utf-8")) - # https://stackoverflow.com/a/6482200 - with open(tmpfile, "w") as outfile: - outfile.write(stdin) - # By default, the file is seemingly created with its execute bit - # unassigned. This is problematic for the RUN instruction. - os.chmod(tmpfile, 0o755) - cli.script = str(tmpfile) + tmp = tempfile.NamedTemporaryFile() + with open(tmp.name, 'w') as fd: + fd.write(stdin) + + cli.script = tmp.name + mode = Modify_Mode.SCRIPT - ch.VERBOSE("shell: %s" % cli.shell) + ch.VERBOSE("modify shell: %s" % cli.shell) + ch.VERBOSE("modify mode: %s" % mode.value) ch.ILLERI("mode: %s" % mode) if (cli.test): @@ -222,20 +219,16 @@ def modify_tree_make_script(src_img, path): [lark.Token('IMAGE_REF', str(src_img))], meta) ], meta)) - if (cli.shell is not None): - df_children.append(im.Tree(lark.Token('RULE', 'shell'), - [lark.Token('STRING_QUOTED', '"%s"' % cli.shell), - lark.Token('STRING_QUOTED', '"-c"') - ],meta)) df_children.append(im.Tree(lark.Token('RULE', 'copy'), [im.Tree(lark.Token('RULE', 'copy_shell'), [lark.Token('WORD', path), lark.Token('WORD', '/ch/script.sh') ], meta) ],meta)) + # FIXME: Add error handling if “cli.shell” doesn’t exist (issue #1913). df_children.append(im.Tree(lark.Token('RULE', 'run'), [im.Tree(lark.Token('RULE', 'run_shell'), - [lark.Token('LINE_CHUNK', '/ch/script.sh')], + [lark.Token('LINE_CHUNK', '%s /ch/script.sh' % cli.shell)], meta) ], meta)) return im.Tree(lark.Token('RULE', 'start'), [im.Tree(lark.Token('RULE','dockerfile'), df_children)], meta) From 2cd6149c45f28f40f24defb34159ebc84fa2b1ae Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Thu, 18 Jul 2024 17:39:59 +0000 Subject: [PATCH 60/65] fix CI --- lib/modify.py | 1 - test/build/55_cache.bats | 13 ++++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/lib/modify.py b/lib/modify.py index 219241007..2d484ba11 100644 --- a/lib/modify.py +++ b/lib/modify.py @@ -184,7 +184,6 @@ def modify_tree_make(src_img, cmds): ], meta)) return im.Tree(lark.Token('RULE', 'start'), [im.Tree(lark.Token('RULE','dockerfile'), df_children)], meta) -# FIXME: Combine with “modify_tree_make”? def modify_tree_make_script(src_img, path): """Temporary(?) analog of “modify_tree_make” for the non-interactive version of “modify” using a script. For the command line: diff --git a/test/build/55_cache.bats b/test/build/55_cache.bats index 22d8e07d3..7de25d4ab 100644 --- a/test/build/55_cache.bats +++ b/test/build/55_cache.bats @@ -1606,12 +1606,12 @@ EOF ch-image modify alpine:3.17 tmpimg "${BATS_TMPDIR}/script.sh" blessed_out=$(cat < '/ch/script.sh' | * RUN.S echo bar | * RUN.S echo foo +| * SHELL ['/bin/sh', '-c'] |/ -* SHELL ['/bin/sh', '-c'] * (alpine+3.17) PULL alpine:3.17 * (root) ROOT EOF @@ -1622,17 +1622,12 @@ EOF [[ $status -eq 0 ]] diff -u <(echo "$blessed_out") <(echo "$output" | treeonly) + ch-image build-cache --reset + ch-image pull alpine:3.17 printf 'echo hello\nexit\n' | ch-image modify -i alpine:3.17 tmpimg blessed_out=$(cat < '/ch/script.sh' -| | * RUN.S echo bar -| | * RUN.S echo foo -| |/ -| * SHELL ['/bin/sh', '-c'] -|/ * (alpine+3.17) PULL alpine:3.17 * (root) ROOT EOF From 20e2d9134b26af4eff8016cd9a88440e80b4c2ed Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Fri, 19 Jul 2024 17:27:11 +0000 Subject: [PATCH 61/65] even more suggestions --- bin/ch-image.py.in | 2 +- doc/ch-image.rst | 12 ++++++------ test/build/50_ch-image.bats | 6 +----- test/force-auto.py.in | 6 ------ 4 files changed, 8 insertions(+), 18 deletions(-) diff --git a/bin/ch-image.py.in b/bin/ch-image.py.in index a7bcc9a44..788d9b86c 100644 --- a/bin/ch-image.py.in +++ b/bin/ch-image.py.in @@ -289,8 +289,8 @@ def main(): sp.add_argument("-S", "--shell", metavar="shell", default="/bin/sh", help="foo") sp.add_argument("image_ref", metavar="IMAGE_REF", help="image to modify") sp.add_argument("out_image", metavar="OUT_IMAGE", help="destination of modified image") - # Optional positional argument? https://stackoverflow.com/a/4480202 sp.add_argument("script", metavar="SCRIPT", help="foo", nargs='?') + # Options “modify” shares with “build”. sp.add_argument("-b", "--bind", metavar="SRC[:DST]", action="append", default=[], help="mount SRC at guest DST (default: same as SRC)") diff --git a/doc/ch-image.rst b/doc/ch-image.rst index e2bbadfd1..2329c2aa6 100644 --- a/doc/ch-image.rst +++ b/doc/ch-image.rst @@ -2183,12 +2183,12 @@ and:: That is, :code:`ch-image` uses :code:`COPY` to put the script inside the image, then runs it. -If :code:`SCRIPT` is not provided and standard input is not a TTY, a script is -read from there instead. In this case, standard input is copied in full to a -file in a temporary directory, which is used as the context. The file’s -modification time is set to 1993-10-21T10:00:00Z and its name to a hash of the -content, so the cache hits if the content is the same and misses if not. That -is, the following are equivalent:: +If :code:`SCRIPT` is not provided, standard input is not a TTY, and :code:`-i` +is not specified, a script is read from stdin instead. In this case, standard +input is copied in full to a file in a temporary directory, which is used as the +context. The file’s modification time is set to 1993-10-21T10:00:00Z and its +name to a hash of the content, so the cache hits if the content is the same and +misses if not. That is, the following are equivalent:: $ ch-image modify foo bar <<'EOF' echo hello world diff --git a/test/build/50_ch-image.bats b/test/build/50_ch-image.bats index 33c5188b3..67c0166c8 100644 --- a/test/build/50_ch-image.bats +++ b/test/build/50_ch-image.bats @@ -998,7 +998,6 @@ EOF # non-interactive, script echo "touch /home/bar" >> "${BATS_TMPDIR}/modify-script.sh" - chmod 755 "${BATS_TMPDIR}/modify-script.sh" ch-image modify alpine:3.17 tmpimg "${BATS_TMPDIR}/modify-script.sh" run ch-run tmpimg -- ls /home echo "$output" @@ -1009,10 +1008,7 @@ EOF ch-image modify alpine:3.17 tmpimg <<'EOF' touch /home/foobar EOF - run ch-run tmpimg -- ls /home - echo "$output" - [[ $status -eq 0 ]] - [[ $output = *'foobar'* ]] + [[ -f "$CH_IMAGE_STORAGE/img/tmpimg/home/foobar" ]] # -c fail run ch-image modify -c 'echo foo' -- alpine:3.17 alpine:3.17 diff --git a/test/force-auto.py.in b/test/force-auto.py.in index d6ec94a33..0184fd2da 100644 --- a/test/force-auto.py.in +++ b/test/force-auto.py.in @@ -129,12 +129,6 @@ class Test(abc.ABC): if (self.force in self.force_excludes): print(f"\n# skip: {self}: --force=%s excluded" % self.force) return - # targeted skip, see issue #1904 - if ((self.base == "quay.io/centos/centos:stream8" and self.run == Run.NEEDED and self.force == "seccomp" and not self.preprep) - or (self.base == "fedora:26" and self.run == Run.NEEDED and self.force == "seccomp" and not self.preprep)): - skip = "skip 'see issue #1904'" - else: - skip = "" # scope if ( (self.scope == Scope.STANDARD or self.run == Run.NEEDED) and self.force != "fakeroot"): From 32efbe44cdf53dd6b4d1a954a411253214859bae Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Tue, 23 Jul 2024 15:23:29 +0000 Subject: [PATCH 62/65] couple more suggestions --- bin/ch-checkns.c | 2 +- bin/ch-run.c | 6 +++--- bin/ch_misc.c | 2 +- bin/ch_misc.h | 2 +- lib/charliecloud.py | 2 +- lib/modify.py | 1 - 6 files changed, 7 insertions(+), 8 deletions(-) diff --git a/bin/ch-checkns.c b/bin/ch-checkns.c index c4d02e8da..3425d0528 100644 --- a/bin/ch-checkns.c +++ b/bin/ch-checkns.c @@ -71,7 +71,7 @@ void fatal_(const char *file, int line, int errno_, const char *str) char *url = "https://github.com/hpc/charliecloud/blob/master/bin/ch-checkns.c"; printf("error: %s: %d: %s\n", file, line, str); printf("errno: %d\nsee: %s\n", errno_, url); - exit(EXIT_CHRUN); + exit(EXIT_MISC_ERR); } int main(int argc, char *argv[]) diff --git a/bin/ch-run.c b/bin/ch-run.c index 89630e9b7..bf3c41934 100644 --- a/bin/ch-run.c +++ b/bin/ch-run.c @@ -445,7 +445,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) #ifdef HAVE_FNM_EXTMATCH exit(0); #else - exit(EXIT_CHRUN); + exit(EXIT_MISC_ERR); #endif } else if (!strcmp(arg, "overlayfs")) { #ifdef HAVE_OVERLAYFS @@ -457,13 +457,13 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) #ifdef HAVE_SECCOMP exit(0); #else - exit(EXIT_CHRUN); + exit(EXIT_MISC_ERR); #endif } else if (!strcmp(arg, "squash")) { #ifdef HAVE_LIBSQUASHFUSE exit(0); #else - exit(EXIT_CHRUN); + exit(EXIT_MISC_ERR); #endif } else if (!strcmp(arg, "tmpfs-xattrs")) { #ifdef HAVE_TMPFS_XATTRS diff --git a/bin/ch_misc.c b/bin/ch_misc.c index 8de09e3c1..4dbc7dbd5 100644 --- a/bin/ch_misc.c +++ b/bin/ch_misc.c @@ -605,7 +605,7 @@ noreturn void msg_fatal(const char *file, int line, int errno_, msgv(LL_FATAL, file, line, errno_, fmt, ap); va_end(ap); - exit(EXIT_CHRUN); + exit(EXIT_MISC_ERR); } /* va_list form of msg(). */ diff --git a/bin/ch_misc.h b/bin/ch_misc.h index 5768d54c4..e2f5dcd47 100644 --- a/bin/ch_misc.h +++ b/bin/ch_misc.h @@ -25,7 +25,7 @@ #define WARNINGS_SIZE (4*1024) /* Exit codes (see also: test/common.bash, lib/build.py). */ -#define EXIT_CHRUN 31 +#define EXIT_MISC_ERR 31 #define EXIT_CMD 49 #define EXIT_SQUASH 84 diff --git a/lib/charliecloud.py b/lib/charliecloud.py index 3f729ed0a..37ceb3283 100644 --- a/lib/charliecloud.py +++ b/lib/charliecloud.py @@ -43,7 +43,7 @@ class Build_Mode(enum.Enum): # ch-run exit codes (see also: bin/ch_misc.h) class Ch_Run_Retcode(enum.Enum): - EXIT_CHRUN = 31 + EXIT_MISC_ERR = 31 EXIT_CMD = 49 EXIT_SQUASH = 84 diff --git a/lib/modify.py b/lib/modify.py index 2d484ba11..7ad6eb8d1 100644 --- a/lib/modify.py +++ b/lib/modify.py @@ -89,7 +89,6 @@ def main(cli_): ch.VERBOSE("modify shell: %s" % cli.shell) ch.VERBOSE("modify mode: %s" % mode.value) - ch.ILLERI("mode: %s" % mode) if (cli.test): exit(0) From f4b1ba74333bf33821afff2b81c3dc69543f32b7 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Wed, 24 Jul 2024 15:11:20 +0000 Subject: [PATCH 63/65] docs + comments --- doc/ch-image.rst | 20 ++++++++++++++++++++ lib/modify.py | 8 +++++--- test/run/ch-run_misc.bats | 1 - 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/doc/ch-image.rst b/doc/ch-image.rst index 2329c2aa6..c9d85a22b 100644 --- a/doc/ch-image.rst +++ b/doc/ch-image.rst @@ -594,6 +594,8 @@ not allowed in Git branch names), and the empty base of everything common instruction :code:`RUN echo foo`. +.. _ch-image_build: + :code:`build` ============= @@ -2139,6 +2141,24 @@ Options: :code:`-S`, :code:`--shell SHELL` Use :code:`SHELL` instead of the default :code:`/bin/sh`. +The following options are shared with :code:`ch-image build`. For more details +about these options, see the section on :ref:`build `. + + :code:`-b`, :code:`--bind SRC[:DST]` + For :code:`RUN` instructions only, bind-mount :code:`SRC` at guest + :code:`DST`. + + :code:`--build-arg KEY[=VALUE]` + Set build-time variable :code:`KEY` defined by :code:`ARG` instruction + to :code:`VALUE`. + + :code:`--force[=MODE]` + Use unprivileged build with root emulation mode :code:`MODE`. + + :code:`--force-cmd=CMD,ARG1[,ARG2...]` + If command :code:`CMD` is found in a :code:`RUN` instruction, add the + comma-separated :code:`ARGs` to it. + :code:`ch-image modify` operates in one of the following three modes. If the mode desired is ambiguous, that is an error. diff --git a/lib/modify.py b/lib/modify.py index 7ad6eb8d1..f79338859 100644 --- a/lib/modify.py +++ b/lib/modify.py @@ -31,9 +31,11 @@ def main(cli_): global cli cli = cli_ - # build.py assumes that global cli comes from the “build” function. If we - # don’t assign these values, the program will fail after trying to access - # them. FIXME: Can partially fix this by adding command line opts. + # CLI opts that “build.py” expects, but that don’t make sense in the context + # of “modify”. We set “parse_only” to “False” because we don’t do any + # parsing, and “context” to the root of the filesystem to ensure that + # necessary files (e.g. the modify script) will always be somewhere in the + # context dir. cli.parse_only = False cli.context = os.path.abspath(os.sep) diff --git a/test/run/ch-run_misc.bats b/test/run/ch-run_misc.bats index e5a325788..f9d93afcb 100644 --- a/test/run/ch-run_misc.bats +++ b/test/run/ch-run_misc.bats @@ -18,7 +18,6 @@ demand-overlayfs () { @test 'relative path to image' { # issue #6 scope full - # shellcheck disable=SC2154 cd "$(dirname "$ch_timg")" && ch-run "$(basename "$ch_timg")" -- /bin/true } From 62fe3e2079312e1de3d7418a0e99aa7f5b357317 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Wed, 24 Jul 2024 18:35:00 +0000 Subject: [PATCH 64/65] comment on log level stuff --- bin/ch-image.py.in | 14 +++++++++----- bin/ch_misc.c | 6 ++++++ doc/ch-image.rst | 6 +++--- lib/modify.py | 3 --- lib/pull.py | 2 +- 5 files changed, 19 insertions(+), 12 deletions(-) diff --git a/bin/ch-image.py.in b/bin/ch-image.py.in index 788d9b86c..e3538ec59 100644 --- a/bin/ch-image.py.in +++ b/bin/ch-image.py.in @@ -283,14 +283,18 @@ def main(): # modify sp = ap.add_parser("modify", "foo") add_opts(sp, modify.main, deps_check=True, stog_init=True) - sp.add_argument("-c", metavar="CMD", action="append", default=[], nargs=1, help="foo") - sp.add_argument("-i", "--interactive", action="store_true", help="foo") - sp.add_argument("-t", "--test", action="store_true", help="foo") - sp.add_argument("-S", "--shell", metavar="shell", default="/bin/sh", help="foo") + sp.add_argument("-c", metavar="CMD", action="append", default=[], nargs=1, + help="Run CMD as though specified by a RUN instruction. Can be repeated.") + sp.add_argument("-i", "--interactive", action="store_true", + help="modify in interactive mode, even if stdin is not a TTY") + sp.add_argument("-S", "--shell", metavar="shell", default="/bin/sh", + help="use SHELL instead of the default /bin/sh") sp.add_argument("image_ref", metavar="IMAGE_REF", help="image to modify") sp.add_argument("out_image", metavar="OUT_IMAGE", help="destination of modified image") sp.add_argument("script", metavar="SCRIPT", help="foo", nargs='?') - # Options “modify” shares with “build”. + # Options “modify” shares with “build”. Note that while we could abstract + # this out to avoid repeated lines, as we do for “common_opts”, we’ve decided + # that the tradeoff in code readability wouldn’t be worth it. sp.add_argument("-b", "--bind", metavar="SRC[:DST]", action="append", default=[], help="mount SRC at guest DST (default: same as SRC)") diff --git a/bin/ch_misc.c b/bin/ch_misc.c index 4dbc7dbd5..17e9b1ea0 100644 --- a/bin/ch_misc.c +++ b/bin/ch_misc.c @@ -592,10 +592,16 @@ void msg_error(const char *file, int line, int errno_, va_list ap; va_start(ap, fmt); + /* We print errors at LL_FATAL because, according to our documentation, + errors are never suppressed. Perhaps we need to rename this log level (see + issue #1914). */ msgv(LL_FATAL, file, line, errno_, fmt, ap); va_end(ap); } +/* Note that msg_fatal doesn’t call msg_error like we do in the Python code + because the variable number of arguments make it easier to simply define + separate functions. */ noreturn void msg_fatal(const char *file, int line, int errno_, const char *fmt, ...) { diff --git a/doc/ch-image.rst b/doc/ch-image.rst index c9d85a22b..98ba00a96 100644 --- a/doc/ch-image.rst +++ b/doc/ch-image.rst @@ -2258,9 +2258,9 @@ Interactive mode iterating a container, it rarely saves time over editing a Dockerfile or shell script. Only use it if you really know what you are doing. -If :code:`SCRIPT` is not provided and standard input *is* a TTY, -:code:`ch-image modify` opens an interactive shell. That is, the following are -roughly equivalent (assuming a terminal):: +If :code:`SCRIPT` is not provided and standard input *is* a TTY, or if +:code:`-i` is specified :code:`ch-image modify` opens an interactive shell. That +is, the following are roughly equivalent (assuming a terminal):: $ ch-image modify foo bar diff --git a/lib/modify.py b/lib/modify.py index f79338859..30f29bfe2 100644 --- a/lib/modify.py +++ b/lib/modify.py @@ -91,9 +91,6 @@ def main(cli_): ch.VERBOSE("modify shell: %s" % cli.shell) ch.VERBOSE("modify mode: %s" % mode.value) - if (cli.test): - exit(0) - if (mode == Modify_Mode.INTERACTIVE): # Interactive case diff --git a/lib/pull.py b/lib/pull.py index 9e52b7d85..24e1714b2 100644 --- a/lib/pull.py +++ b/lib/pull.py @@ -106,7 +106,7 @@ def download(self): # currently, this error is only raised if we’ve downloaded the # skinny manifest. have_skinny = True - if (ch.arch in "amd64"): + if (ch.arch == "amd64"): # We’re guessing that enough arch-unaware images are amd64 to # barge ahead if requested architecture is amd64. ch.arch = "yolo" From 621b19a52f619909a510562660c60e6abc620aad Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Wed, 24 Jul 2024 19:47:33 +0000 Subject: [PATCH 65/65] some cleanup --- lib/modify.py | 8 +--- test/build/50_ch-image.bats | 6 +-- test/common.bash | 2 +- test/run/ch-run_escalated.bats | 6 +-- test/run/ch-run_join.bats | 36 ++++++++--------- test/run/ch-run_misc.bats | 70 +++++++++++++++++----------------- 6 files changed, 61 insertions(+), 67 deletions(-) diff --git a/lib/modify.py b/lib/modify.py index 30f29bfe2..3a59b9459 100644 --- a/lib/modify.py +++ b/lib/modify.py @@ -21,12 +21,6 @@ class Modify_Mode(enum.Enum): SCRIPT = "script" def main(cli_): - global called - called = True - - # Need to pass tree to build.py - global tree - # In this file, “cli” is used as a global variable global cli cli = cli_ @@ -200,7 +194,7 @@ def modify_tree_make_script(src_img, path): WORD /path/to/script WORD /ch/script.sh run run_shell - LINE_CHUNK /ch/script.sh + LINE_CHUNK /bin/sh /ch/script.sh """ # Children of dockerfile tree df_children = [] diff --git a/test/build/50_ch-image.bats b/test/build/50_ch-image.bats index 67c0166c8..290e929b1 100644 --- a/test/build/50_ch-image.bats +++ b/test/build/50_ch-image.bats @@ -864,17 +864,17 @@ EOF @test 'ch-run storage errors' { run ch-run -v -w alpine:3.17 -- /bin/true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *'error: --write invalid when running by name'* ]] run ch-run -v "$CH_IMAGE_STORAGE"/img/alpine+3.17 -- /bin/true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *"error: can't run directory images from storage (hint: run by name)"* ]] run ch-run -v -s /doesnotexist alpine:3.17 -- /bin/true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *'warning: storage directory not found: /doesnotexist'* ]] [[ $output = *"error: can't stat: alpine:3.17: No such file or directory"* ]] } diff --git a/test/common.bash b/test/common.bash index beb948051..3e7fa5be0 100644 --- a/test/common.bash +++ b/test/common.bash @@ -330,7 +330,7 @@ export BATS_TMPDIR=$btnew [[ $(stat -c %a "$BATS_TMPDIR") = '700' ]] # ch-run exit codes. (see also: ch_misc.h, lib/build.py) -CH_ERR_RUN=31 +CH_ERR_MISC=31 CH_ERR_CMD=49 #CH_ERR_SQUASH=84 # Currently not used diff --git a/test/run/ch-run_escalated.bats b/test/run/ch-run_escalated.bats index 548084e39..6501104b7 100644 --- a/test/run/ch-run_escalated.bats +++ b/test/run/ch-run_escalated.bats @@ -15,7 +15,7 @@ load ../common [[ -g $ch_run_tmp ]] run "$ch_run_tmp" --version echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *': please report this bug ('* ]] rm "$ch_run_tmp" } @@ -32,7 +32,7 @@ load ../common [[ -u $ch_run_tmp ]] run "$ch_run_tmp" --version echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *': please report this bug ('* ]] sudo rm "$ch_run_tmp" } @@ -71,7 +71,7 @@ load ../common fi run sudo -u root -g "$(id -gn)" "$ch_runfile" -v --version echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *'please report this bug ('* ]] } diff --git a/test/run/ch-run_join.bats b/test/run/ch-run_join.bats index 7521d3875..0ff9c07cd 100644 --- a/test/run/ch-run_join.bats +++ b/test/run/ch-run_join.bats @@ -267,36 +267,36 @@ unset_vars () { # --join but no join count run ch-run --join "$ch_timg" -- true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output =~ 'join: no valid peer group size found' ]] ipc_clean_p # join count no digits run ch-run --join-ct=a "$ch_timg" -- true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output =~ 'join-ct: no digits found' ]] SLURM_CPUS_ON_NODE=a run ch-run --join "$ch_timg" -- true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output =~ 'SLURM_CPUS_ON_NODE: no digits found' ]] ipc_clean_p # join count empty string run ch-run --join-ct='' "$ch_timg" -- true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output =~ '--join-ct: no digits found' ]] SLURM_CPUS_ON_NODE=-1 run ch-run --join "$ch_timg" -- true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output =~ 'join: no valid peer group size found' ]] ipc_clean_p # --join-ct digits followed by extra goo (OK from environment variable) run ch-run --join-ct=1a "$ch_timg" -- true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output =~ '--join-ct: extra characters after digits' ]] ipc_clean_p @@ -306,48 +306,48 @@ unset_vars () { # join count above INT_MAX run ch-run --join-ct=2147483648 "$ch_timg" -- true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output =~ $range_re ]] SLURM_CPUS_ON_NODE=2147483648 \ run ch-run --join "$ch_timg" -- true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output =~ $range_re ]] ipc_clean_p # join count below INT_MIN run ch-run --join-ct=-2147483649 "$ch_timg" -- true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output =~ $range_re ]] SLURM_CPUS_ON_NODE=-2147483649 \ run ch-run --join "$ch_timg" -- true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output =~ $range_re ]] ipc_clean_p # join count above LONG_MAX run ch-run --join-ct=9223372036854775808 "$ch_timg" -- true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output =~ $range_re ]] SLURM_CPUS_ON_NODE=9223372036854775808 \ run ch-run --join "$ch_timg" -- true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output =~ $range_re ]] ipc_clean_p # join count below LONG_MIN run ch-run --join-ct=-9223372036854775809 "$ch_timg" -- true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output =~ $range_re ]] SLURM_CPUS_ON_NODE=-9223372036854775809 \ run ch-run --join "$ch_timg" -- true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output =~ $range_re ]] ipc_clean_p } @@ -361,11 +361,11 @@ unset_vars () { # join tag empty string run ch-run --join-tag='' "$ch_timg" -- true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output =~ 'join: peer group tag cannot be empty string' ]] SLURM_STEP_ID='' run ch-run --join "$ch_timg" -- true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output =~ 'join: peer group tag cannot be empty string' ]] ipc_clean_p } @@ -466,14 +466,14 @@ unset_vars () { # Can’t join namespaces of processes we don’t own. run ch-run -v --join-pid=1 "$ch_timg" -- true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *"join: can't open /proc/1/ns/user: Permission denied"* ]] # Can’t join namespaces of processes that don’t exist. pid=2147483647 run ch-run -v --join-pid="$pid" "$ch_timg" -- true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *"join: no PID ${pid}: /proc/${pid}/ns/user not found"* ]] } diff --git a/test/run/ch-run_misc.bats b/test/run/ch-run_misc.bats index f9d93afcb..e33941eea 100644 --- a/test/run/ch-run_misc.bats +++ b/test/run/ch-run_misc.bats @@ -121,7 +121,7 @@ EOF run ch-run --home "$ch_timg" -- /bin/sh -c 'echo $HOME' export HOME="$home_tmp" echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] # shellcheck disable=SC2016 [[ $output = *'--home failed: $HOME not set'* ]] @@ -132,7 +132,7 @@ EOF run ch-run --home "$ch_timg" -- /bin/sh -c 'echo $HOME' export USER=$user_tmp echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] # shellcheck disable=SC2016 [[ $output = *'$USER not set'* ]] } @@ -212,7 +212,7 @@ EOF # Error if directory does not exist. run ch-run --cd /goops "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output =~ "can't cd to /goops: No such file or directory" ]] } @@ -321,112 +321,112 @@ EOF # empty argument to --bind run ch-run -b '' "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *'--bind: no source provided'* ]] # source not provided run ch-run -b :/mnt/9 "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *'--bind: no source provided'* ]] # destination not provided run ch-run -b "${bind1_dir}:" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *'--bind: no destination provided'* ]] # destination is / run ch-run -b "${bind1_dir}:/" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *"--bind: destination can't be /"* ]] # destination is relative run ch-run -b "${bind1_dir}:foo" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *"--bind: destination must be absolute"* ]] # destination climbs out of image, exists run ch-run -b "${bind1_dir}:/.." "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *"can't bind: "*"/${USER}.ch not subdirectory of "*"/${USER}.ch/mnt"* ]] # destination climbs out of image, does not exist run ch-run -b "${bind1_dir}:/../doesnotexist/a" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *"can't mkdir: "*"/${USER}.ch/doesnotexist not subdirectory of "*"/${USER}.ch/mnt"* ]] [[ ! -e ${ch_imgdir}/doesnotexist ]] # source does not exist run ch-run -b "/doesnotexist" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *"can't bind: source not found: /doesnotexist"* ]] # destination does not exist and image is not writeable run ch-run -b "${bind1_dir}:/doesnotexist" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *"can't mkdir: "*"/${USER}.ch/mnt/doesnotexist: Read-only file system"* ]] # neither source nor destination exist run ch-run -b /doesnotexist-out:/doesnotexist-in "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *"can't bind: source not found: /doesnotexist-out"* ]] # correct bind followed by source does not exist run ch-run -b "${bind1_dir}:/mnt/0" -b /doesnotexist "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *"can't bind: source not found: /doesnotexist"* ]] # correct bind followed by destination does not exist run ch-run -b "${bind1_dir}:/mnt/0" -b "${bind2_dir}:/doesnotexist" \ "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *"can't mkdir: "*"/${USER}.ch/mnt/doesnotexist: Read-only file system"* ]] # destination is broken symlink run ch-run -b "${bind1_dir}:/mnt/link-b0rken-abs" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *"can't mkdir: symlink not relative: "*"/${USER}.ch/mnt/mnt/link-b0rken-abs"* ]] # destination is absolute symlink outside image run ch-run -b "${bind1_dir}:/mnt/link-bad-abs" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *"can't bind: "*" not subdirectory of"* ]] # destination relative symlink outside image run ch-run -b "${bind1_dir}:/mnt/link-bad-rel" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *"can't bind: "*" not subdirectory of"* ]] # mkdir(2) under existing bind-mount, default, first level run ch-run -b "${bind1_dir}:/proc/doesnotexist" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *"can't mkdir: "*"/${USER}.ch/mnt/proc/doesnotexist under existing bind-mount "*"/${USER}.ch/mnt/proc "* ]] # mkdir(2) under existing bind-mount, user-supplied, first level run ch-run -b "${bind1_dir}:/mnt/0" \ -b "${bind2_dir}:/mnt/0/foo" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *"can't mkdir: "*"/${USER}.ch/mnt/mnt/0/foo under existing bind-mount "*"/${USER}.ch/mnt/mnt/0 "* ]] # mkdir(2) under existing bind-mount, default, 2nd level run ch-run -b "${bind1_dir}:/proc/sys/doesnotexist" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *"can't mkdir: "*"/${USER}.ch/mnt/proc/sys/doesnotexist under existing bind-mount "*"/${USER}.ch/mnt/proc "* ]] } @@ -623,13 +623,13 @@ EOF # file does not exist run ch-run --set-env=doesnotexist.txt "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *"can't open: doesnotexist.txt: No such file or directory"* ]] # /ch/environment missing run ch-run --set-env "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *"can't open: /ch/environment: No such file or directory"* ]] # Note: I’m not sure how to test an error during reading, i.e., getline(3) @@ -639,14 +639,14 @@ EOF echo 'FOO bar' > "$f_in" run ch-run --set-env="$f_in" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *"can't parse variable: no delimiter: ${f_in}:1"* ]] # invalid line: no name echo '=bar' > "$f_in" run ch-run --set-env="$f_in" "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *"can't parse variable: empty name: ${f_in}:1"* ]] } @@ -665,7 +665,7 @@ EOF # missing environment variable run ch-run --set-env='$PATH:foo' "$ch_timg" -- /bin/true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *'$PATH:foo: No such file or directory'* ]] } @@ -712,7 +712,7 @@ EOF printf '\n# Empty string\n\n' run ch-run --unset-env= "$ch_timg" -- env echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *'--unset-env: GLOB must have non-zero length'* ]] } @@ -915,7 +915,7 @@ EOF # image is file but not sqfs run ch-run -vv ./fixtures/README -- /bin/true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output = *'magic expected: 6873 7173; actual: 596f 7520'* ]] [[ $output = *'unknown image type: '*'/fixtures/README'* ]] @@ -962,7 +962,7 @@ EOF run ch-run "$img" -- /bin/true touch "${img}/${f}" # restore before test fails for idempotency echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] r="can't bind: destination not found: .+/${f}" echo "expected: ${r}" [[ $output =~ $r ]] @@ -988,7 +988,7 @@ EOF rmdir "${img}/${f}" # restore before test fails for idempotency touch "${img}/${f}" echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] r="can't bind .+ to /.+/${f}: Not a directory" echo "expected: ${r}" [[ $output =~ $r ]] @@ -1001,7 +1001,7 @@ EOF run ch-run "$img" -- /bin/true mkdir "${img}/${d}" # restore before test fails for idempotency echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] r="can't bind: destination not found: .+/${d}" echo "expected: ${r}" [[ $output =~ $r ]] @@ -1016,7 +1016,7 @@ EOF rm "${img}/${d}" # restore before test fails for idempotency mkdir "${img}/${d}" echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] r="can't bind .+ to /.+/${d}: Not a directory" echo "expected: ${r}" [[ $output =~ $r ]] @@ -1027,7 +1027,7 @@ EOF run ch-run --private-tmp "$img" -- /bin/true mkdir "${img}/tmp" # restore before test fails for idempotency echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] r="can't mount tmpfs at /.+/tmp: No such file or directory" echo "expected: ${r}" [[ $output =~ $r ]] @@ -1165,7 +1165,7 @@ EOF # failure at quiet level 3 run ch-run -qqq --test=log-fail - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output != *'info'* ]] [[ $output != *'warning: warning'* ]] [[ $output = *'error: the program failed inexplicably'* ]] @@ -1177,6 +1177,6 @@ EOF # bad tmpfs size run ch-run --write-fake=foo "$ch_timg" -- true echo "$output" - [[ $status -eq $CH_ERR_RUN ]] + [[ $status -eq $CH_ERR_MISC ]] [[ $output == *'cannot mount tmpfs for overlay: Invalid argument'* ]] }