improve install/update semantics

giteapc.py

@@ -36,6 +36,16 @@ is either a full name like "Lephenixnoir/sh-elf-gcc", or a short name like
 {R}giteapc list{_} [{R}-r{_}] [{g}{i}PATTERN{_}]
   List all repositories on this computer. With -r, list repositories on the
   forge. A wildcard pattern can be specified to filter the results.
+{R}giteapc install{_} [{R}--https{_}|{R}--ssh{_}] [{R}-u{_}] [{R}-y{_}] [{R}-n{_}] {g}{i}REPOSITORY{_}[{R}@{_}{g}{i}VERSION{_}][{R}:{_}{g}{i}CONFIG{_}]{g}{i}...{_}
+  Fetch repositories and their dependencies, then build and install them.
+  With -u, pulls local repositories (update mode). With -y, do not ask for
+  interactive confirmation. With -n, don't build or install (dry run).
+{R}giteapc uninstall{_} [{R}-k{_}] {g}{i}REPOSITORY...{_}
+  Uninstall the build products of the specified repositories and remove the
+  source files. With -k, keep the source files.
+
+Advanced commands:
+
 {R}giteapc fetch{_} [{R}--https{_}|{R}--ssh{_}] [{R}-u{_}] [{R}-f{_}] [{g}{i}REPOSITORY{_}[{R}@{_}{g}{i}VERSION{_}]{g}{i}...{_}]
   Clone, fetch, or pull a repository. If no repository is specified, fetches
   all local repositories. HTTPS or SSH can be selected when cloning (HTTPS by
@@ -44,18 +54,10 @@ is either a full name like "Lephenixnoir/sh-elf-gcc", or a short name like
   Show the branches and tags (versions) for the specified local repositories.
   With -r, show information for remote repositories on the forge.
   With -p, just print the path of local repositories (useful in scripts).
-{R}giteapc build{_} [{R}-i{_}] [{R}-u{_}] [{R}--skip-configure{_}] {g}{i}REPOSITORY{_}[{R}@{_}{g}{i}VERSION{_}][{R}:{_}{g}{i}CONFIG{_}]{g}{i}...{_}
+{R}giteapc build{_} [{R}-i{_}] [{R}--skip-configure{_}] {g}{i}REPOSITORY{_}[{R}@{_}{g}{i}VERSION{_}][{R}:{_}{g}{i}CONFIG{_}]{g}{i}...{_}
   Configure and build a local repository. A specific configuration can be
   requested. With -i, also install if build is successful. --skip-configure
-  builds without configuring (useful for rebuilds). With -u, pull the current
+  builds without configuring (useful for rebuilds).
-  branch before building (update mode).
-{R}giteapc install{_} [{R}--https{_}|{R}--ssh{_}] [{R}-u{_}] [{R}-y{_}] {g}{i}REPOSITORY{_}[{R}@{_}{g}{i}VERSION{_}][{R}:{_}{g}{i}CONFIG{_}]{g}{i}...{_}
-  Fetch repositories and their dependencies, then build and install them.
-  With -u, pulls local repositories (update mode). With -yes, do not ask for
-  interactive confirmation.
-{R}giteapc uninstall{_} [{R}-k{_}] {g}{i}REPOSITORY...{_}
-  Uninstall the build products of the specified repositories and remove the
-  source files. With -k, keep the source files.
 
 {W}Important folders{_}
 
@@ -88,12 +90,12 @@ commands = {
     },
     "build": {
         "function": giteapc.repo.build,
-        "args": "install:-i,--install skip_configure:--skip-configure "+\
+        "args": "install:-i,--install skip_configure:--skip-configure",
-                "update:-u,--update",
     },
     "install": {
         "function": giteapc.repo.install,
-        "args": "use_ssh:--ssh use_https:--https update:-u,--update",
+        "args": "use_ssh:--ssh use_https:--https update:-u,--update "+\
+                "yes:-y,--yes dry_run:-n,--dry-run",
     },
     "uninstall": {
         "function": giteapc.repo.uninstall,

giteapc/repo.py

@@ -99,21 +99,48 @@ def pretty_repo(r):
     color = "ARGYBMW"[sum(map(ord,r.owner[:5])) % 7]
     return colors()[color] + "{}{_}/{W}{}{_}".format(r.owner,r.name,**colors())
 
-def split_config(name):
+#
-    """Splits REPOSITORY[@VERSION][:CONFIGURATION] into components."""
+# Repository specifications
-    RE_CONFIG = re.compile(r'^([^@:]+)(?:@([^@:]+))?(?:[:]([^@:]+))?')
+#
-    m = re.match(RE_CONFIG, name)
+
+class Spec:
+    # A spec in REPOSITORY[@VERSION][:CONFIGURATION]
+    RE_SPEC = re.compile(r'^([^@:]+)(?:@([^@:]+))?(?:[:]([^@:]+))?')
+
+    def __init__(self, string):
+        m = re.match(Spec.RE_SPEC, string)
         if m is None:
-        return None
+            raise Error(f"wrong format in specification {string}")
 
-    repo, version, config = m[1], m[2] or "", m[3] or ""
+        self.name    = m[1]
-    return repo, version, config
+        self.version = m[2]
+        self.config  = m[3]
+        self.repo    = None
 
-def make_config(name, version, config):
+    def resolve(self, local_only=False, remote_only=False):
-    version = f"@{version}" if version else ""
+        self.repo = resolve(self.name, local_only, remote_only)
-    config  = f":{config}"  if config  else ""
+        return self.repo
+
+    def str(self, pretty=False):
+        if self.repo and pretty:
+            name = pretty_repo(self.repo)
+        elif self.repo:
+            name = self.repo.fullname
+        else:
+            name = self.name
+
+        version = f"@{self.version}" if self.version else ""
+        config  = f":{self.config}"  if self.config  else ""
         return name + version + config
 
+    def __str__(self):
+        return self.str(pretty=False)
+    def __repr__(self):
+        return self.str(pretty=False)
+
+    def pretty_str(self):
+        return self.str(pretty=True)
+
 #
 # repo list command
 #
@@ -174,25 +201,17 @@ def fetch(*args, use_ssh=False, use_https=False, force=False, update=False):
                 r.pull()
         return 0
 
-    for spec in args:
+    for arg in args:
-        name, version, config = split_config(spec)
+        s = Spec(arg)
-        r = resolve(name)
+        r = s.resolve()
 
         # If this is a local repository, just git fetch
         if not r.remote:
-            if version:
-                msg("Checking out {W}{}{_}".format(version, **colors()))
-                r.checkout(version)
-
             msg(f"Fetching {pretty_repo(r)}...")
             r.fetch()
-            if update:
+        # If this is a remote repository, clone it
-                r.pull()
+        else:
-            continue
-
             msg(f"Cloning {pretty_repo(r)}...")
-
-        # For remote repositories, make sure the repository supports GiteaPC
             has_tag = "giteapc" in gitea.repo_topics(r)
 
             if has_tag or force:
@@ -200,7 +219,15 @@ def fetch(*args, use_ssh=False, use_https=False, force=False, update=False):
             if not has_tag and force:
                 warn(f"{r.fullname} doesn't have the [giteapc] tag")
             if not has_tag and not force:
-            fatal(f"{r.fullname} doesn't have the [giteapc] tag, use -f to force")
+                return fatal(f"{r.fullname} doesn't have the [giteapc] tag, "+\
+                    "use -f to force")
+
+        # Checkout requested version, if any
+        if s.version:
+            msg("Checking out {W}{}{_}".format(s.version, **colors()))
+            r.checkout(s.version)
+        if update:
+            r.pull()
 
 #
 # repo show command
@@ -234,38 +261,31 @@ def show(*args, remote=False, path=False):
 # repo build command
 #
 
-def build(*args, install=False, skip_configure=False, update=False):
+def build(*args, install=False, skip_configure=False):
     if len(args) < 1:
         return fatal("repo build: specify at least one repository")
 
     specs = []
-    for spec in args:
+    for arg in args:
-        repo, version, config = split_config(spec)
+        s = arg if isinstance(arg, Spec) else Spec(arg)
-        repo = resolve(repo, local_only=True)
+        s.resolve(local_only=True)
-        specs.append((repo, version, config))
+        specs.append(s)
 
-    for (r, version, config) in specs:
+    for s in specs:
+        r = s.repo
         pretty = pretty_repo(r)
-        config_string = f" for {config}" if config else ""
+        config_string = f" for {s.config}" if s.config else ""
 
-        if version != "":
+        if s.version:
-            msg("{}: Checking out {W}{}{_}".format(pretty, version, **colors()))
+            msg("{}: Checking out {W}{}{_}".format(pretty,s.version,**colors()))
-            r.checkout(version)
+            r.checkout(s.version)
-        if update:
-            previous = r.describe()
-            r.pull()
-            new = r.describe()
-            if new == previous:
-                msg("{}: Still at {W}{}{_}, skipping build".format(pretty,
-                    previous, **colors()))
-                continue
 
         # Check that the project has a Makefile
         if not os.path.exists(r.makefile):
             raise Error(f"{r.fullname} has no giteapc.make")
 
         env = os.environ.copy()
-        if config:
+        if s.config:
             env["GITEAPC_CONFIG"] = config
         env["GITEAPC_PREFIX"] = PREFIX_FOLDER
 
@@ -287,59 +307,65 @@ def build(*args, install=False, skip_configure=False, update=False):
 # repo install command
 #
 
-def install(*args, use_https=False, use_ssh=False, update=False, yes=False):
+def search_dependencies(names, fetched, plan, **kwargs):
-    if args == ():
+    for name in names:
+        s = Spec(name)
+        r = s.resolve()
+
+        if r.fullname not in fetched:
+            fetch(r.fullname, **kwargs)
+            fetched.add(r.fullname)
+            # Schedule dependencies before r
+            search_dependencies(r.dependencies(), fetched, plan, **kwargs)
+            plan.append(s)
+
+def install(*args, use_https=False, use_ssh=False, update=False, yes=False,
+    dry_run=False):
+
+    # Update all repositories
+    if args == () and update == True:
+        args = [ r.fullname for r in LocalRepo.all() ]
+
+    # Fetch every repository and determine its dependencies to form a basic
+    # plan of what to build in what order
+
+    basic_plan = []
+    search_dependencies(args, set(), basic_plan, use_https=use_https,
+        use_ssh=use_ssh, update=update)
+
+    # Sanitize the build plan by checking occurrences of the same repository
+    # are consistent and eliminating duplicates
+
+    # Build plan organized by name
+    named = dict()
+    # Final plan
+    plan = []
+
+    for s in basic_plan:
+        r = s.repo
+        if r.fullname not in named:
+            named[r.fullname] = s
+            plan.append(s)
+            continue
+        s2 = named[r.fullname]
+        if not s2.compatible_with(s):
+            return fatal(f"repo install: cannot install both {s} and {s2}")
+
+    # Plan review and confirmation
+
+    msg("Will install:", ", ".join(s.pretty_str() for s in plan))
+
+    if dry_run:
         return 0
 
-    def recursive_fetch(spec, fetched_so_far):
-        repos_to_build = []
-        repo, version, config = split_config(spec)
-        r = resolve(repo, local_only=True)
-
-        if r.fullname not in fetched_so_far:
-            fetched_so_far.add(r.fullname)
-            fetch(repo, use_https=use_https, use_ssh=use_ssh, update=update)
-
-            for dep_spec in r.dependencies():
-                rtb, fsf = recursive_fetch(dep_spec, fetched_so_far)
-                repos_to_build += rtb
-                fetched_so_far = fetched_so_far.union(fsf)
-
-        return repos_to_build + [(r, version, config)], fetched_so_far
-
-    # First download every repository, and only then build
-    repos_to_build = []
-    fetched_so_far = set()
-
-    for spec in args:
-        rtb, fsf = recursive_fetch(spec, fetched_so_far)
-        repos_to_build += rtb
-        fetched_so_far = fetched_so_far.union(fsf)
-
-    # Eliminate duplicates and look for version collisions
-    rd = dict()
-
-    for (r, version, config) in repos_to_build:
-        name = r.fullname
-        if name not in rd:
-            rd[name] = (version, config)
-        elif rd[name] != (version, config):
-            s1 = make_config(name, *rd[name])
-            s2 = make_config(name, version, config)
-            return fatal(f"repo install: cannot install both {s1} and {s2}")
-
-    msg("Will install:", ", ".join(make_config(pretty_repo(r), version, config)
-        for (r, version, config) in repos_to_build))
-
     if not yes:
         msg("Is that okay (Y/n)? ", end="")
         confirm = input().strip()
         if confirm not in ["Y", "y", ""]:
             return 1
 
-    for (r, version, config) in repos_to_build:
+    # Final build
-        build(make_config(r.fullname, version, config), install=True,
+    build(*plan, install=True)
-            update=update)
 
 #
 # repo uninstall command