fxconv: support metadata discovery in fxconv-metadata.txt

This change adds a new way for fxconv to discover metadata for file
conversions. This complements the existing mechanism of passing
parameters on the command-line.

The new mechanism activates when fxconv is called without a type
argument. Type information and metadata are searched in an
fxconv-metadata.txt file in the same folder as the resource. The
metadata file lists parameters, with some additional flexibility enabled
by the use of wildcards.

This way of declaring will replace command-line argument passing, which
currently read parameters from the unreadable and not-so-maitainable
project.cfg file. Both the GNU make and CMake build systems should use
it in the future. The current way is still supported only for older
projects and one-shot conversions outside of projects.

Makefile

@@ -41,14 +41,12 @@ all: $(bin)
 
 all-fxsdk: bin/fxsdk.sh
 all-fxg1a: bin/fxg1a
-all-fxconv: bin/fxconv-main.py
+all-fxconv:
 
 # Explicit targets
 
 bin/fxsdk.sh: fxsdk/fxsdk.sh | bin/
 	sed $(sed) $< > $@
-bin/fxconv-main.py: fxconv/fxconv-main.py | bin/
-	sed $(sed) $< > $@
 bin/fxg1a: $(obj-fxg1a) | bin/
 	gcc $^ -o $@ $(lflags)
 
@@ -98,7 +96,7 @@ install: $(bin)
 	install -d $(PREFIX)/share/fxsdk/assets
 	install fxsdk/assets/* $(m644) $(PREFIX)/share/fxsdk/assets
 	install bin/fxsdk.sh $(m755) $(PREFIX)/bin/fxsdk
-	install bin/fxconv-main.py $(m755) $(PREFIX)/bin/fxconv
+	install fxconv/fxconv-main.py $(m755) $(PREFIX)/bin/fxconv
 	install fxconv/fxconv.py $(m644) $(PREFIX)/bin
 
 uninstall:

fxconv/fxconv-main.py

@@ -3,59 +3,98 @@
 import getopt
 import sys
 import os
+import re
+import fnmatch
 import fxconv
-import subprocess
-
-# Note: this line is edited at compile time to insert the install folder
-PREFIX="""\
-""".strip()
 
 help_string = f"""
-usage: fxconv [-s] <python script> [files...]
+usage: fxconv [<TYPE>] <INPUT> -o <OUTPUT> [--fx|--cg] [<PARAMETERS>...]
-       fxconv  -b  <bin file> -o <object file> [parameters...]
-       fxconv  -i  <png file> -o <object file> (--fx|--cg) [parameters...]
-       fxconv  -f  <png file> -o <object file> [parameters...]
 
-fxconv converts data files such as images and fonts into gint formats
+fxconv converts resources such as images and fonts into binary formats for
-optimized for fast execution, or into object files.
+fxSDK applications, using gint and custom conversion formats.
 
-Operating modes:
+When no TYPE is specified (automated mode), fxconv looks for type and
-  -s, --script    Expose the fxconv module and run this Python script
+parameters in an fxconv-metadata.txt file in the same folder as the input. This
-  -b, --binary    Turn data into an object file, no conversion
+is normally the default for add-ins.
-  -i, --image     Convert to gint's bopti image format
+
+When TYPE is specified (one-shot conversion), it should be one of:
+  -b, --binary    Turn data into an object file without conversion
   -f, --font      Convert to gint's topti font format
+  --bopti-image   Convert to gint's bopti image format
   --libimg-image  Convert to the libimg image format
+  --custom        Use converters.py; you might want to specify an explicit type
+                  by adding a parameter type:your_custom_type (see below)
 
-When using -s, additional arguments are stored in the [fxconv.args] variable of
+During one-shot conversions, parameters can be specified with a "NAME:VALUE"
-the module. This is intended to be a restricted list of file names specified by
+syntax (names can contain dots). For example:
-a Makefile, used to convert only a subset of the files in the script.
-
-The operating mode options are shortcuts to convert single files without a
-script. They accept parameters with a "category.key:value" syntax, for example:
   fxconv -f myfont.png -o myfont.o charset:ascii grid.padding:1 height:7
 
-When converting images, use --fx (black-and-white calculators) or --cg (16-bit
+Some formats differ between platforms so you should specify it when possible:
-color calculators) to specify the target machine.
+  --fx, --fx9860G  Casio fx-9860G family (black-and-white calculators)
-
+  --cg, --fxCG50   Casio fx-CG 50 family (16-bit color calculators)
-Install PREFIX is set to '{PREFIX}'.
 """.strip()
 
 # Simple error-warnings system
+FxconvError = fxconv.FxconvError
+
 def err(msg):
 	print("\x1b[31;1merror:\x1b[0m", msg, file=sys.stderr)
+	return 1
 def warn(msg):
 	print("\x1b[33;1mwarning:\x1b[0m", msg, file=sys.stderr)
 
-# "converters" module from the user project
+# "converters" module from the user project... if it exists
 try:
 	import converters
 except ImportError:
 	converters = None
 
+def parse_parameters(params):
+	"""Parse parameters of the form "NAME:VALUE" into a dictionary."""
+	d = dict()
+
+	def insert(d, path, value):
+		if len(path) == 1:
+			d[path[0]] = value
+		else:
+			if not path[0] in d:
+				d[path[0]] = dict()
+			insert(d[path[0]], path[1:], value)
+
+	for decl in params:
+		if ":" not in decl:
+			raise FxconvError(f"invalid parameter {decl}, ignoring")
+		else:
+			name, value = decl.split(":", 1)
+			insert(d, name.split("."), value.strip())
+
+	return d
+
+def parse_parameters_metadata(contents):
+	"""Parse parameters from a metadata file contents."""
+
+	RE_COMMENT = re.compile(r'#.*$', re.MULTILINE)
+	contents = re.sub(RE_COMMENT, "", contents)
+
+	RE_WILDCARD = re.compile(r'^(\S(?:[^:\s]|\\:|\\ )*)\s*:\s*$', re.MULTILINE)
+	lead, *elements = [ s.strip() for s in re.split(RE_WILDCARD, contents) ]
+
+	if lead:
+		raise FxconvError(f"invalid metadata: {lead} appears before wildcard")
+
+	# Group elements by pairs (left: wildcard, right: list of properties)
+	elements = list(zip(elements[::2], elements[1::2]))
+
+	metadata = []
+	for (wildcard, params) in elements:
+		params = [ s.strip() for s in params.split("\n") if s.strip() ]
+		metadata.append((wildcard, parse_parameters(params)))
+
+	return metadata
+
 def main():
-	# Default execution mode is to run a Python script for conversion
+	types = "binary image font bopti-image libimg-image custom"
-	modes = "script binary image font bopti-image libimg-image"
+	mode = ""
-	mode = "s"
 	output = None
 	model = None
 	target = { 'toolchain': None, 'arch': None, 'section': None }
@@ -68,24 +107,25 @@ def main():
 		sys.exit(1)
 
 	try:
-		longs = "help output= fx cg toolchain= arch= section= custom " + modes
+		models = "fx cg fx9860G fxCG50"
+		longs = f"help output= toolchain= arch= section= {models} {types}"
 		opts, args = getopt.gnu_getopt(sys.argv[1:], "hsbifo:", longs.split())
 	except getopt.GetoptError as error:
-		err(error)
+		return err(error)
-		sys.exit(1)
 
 	for name, value in opts:
 		# Print usage
 		if name == "--help":
-			err(help_string, file=sys.stderr)
+			print(help_string, file=sys.stderr)
-			sys.exit(0)
+			return 0
-		# TODO: fxconv: verbose mode
-		elif name == "--verbose":
-			pass
 		elif name in [ "-o", "--output" ]:
 			output = value
 		elif name in [ "--fx", "--cg" ]:
 			model = name[2:]
+		elif name == "--fx9860G":
+			model = "fx"
+		elif name == "--fxCG50":
+			model = "cg"
 		elif name == "--toolchain":
 			target['toolchain'] = value
 		elif name == "--arch":
@@ -101,54 +141,33 @@ def main():
 
 	# Remaining arguments
 	if args == []:
-		err(f"execution mode -{mode} expects an input file")
+		err(f"no input file")
 		sys.exit(1)
 	input = args.pop(0)
 
-	# In --script mode, run the Python script with an augmented PYTHONPATH
+	# In automatic mode, look for information in fxconv-metadata.txt
+	if mode == "":
+		metadata_file = os.path.dirname(input) + "/fxconv-metadata.txt"
+		basename = os.path.basename(input)
 
-	if mode == "s":
+		if not os.path.exists(metadata_file):
-		if output is not None:
+			return err(f"using auto mode but {metadata_file} does not exist")
-			warn("option --output is ignored in script mode")
 
-		if PREFIX == "":
+		with open(metadata_file, "r") as fp:
-			err("unknown or invalid install path x_x")
+			metadata = parse_parameters_metadata(fp.read())
-			sys.exit(1)
 
-		env = os.environ.copy()
+			params = dict()
-		if "PYTHONPATH" in env:
+			for (wildcard, p) in metadata:
-			env["PYTHONPATH"] += f":{PREFIX}/bin"
+				if fnmatch.fnmatchcase(basename, wildcard):
-		else:
+					params.update(**p)
-			env["PYTHONPATH"] = f"{PREFIX}/bin"
-
-		p = subprocess.run([ sys.executable, input ], env=env)
-		if p.returncode != 0:
-			sys.exit(1)
-
-	# In shortcut conversion modes, read parameters from the command-line
 
+	# In manual conversion modes, read parameters from the command-line
 	else:
-		def check(arg):
+		params = parse_parameters(args)
-			if ':' not in arg:
-				warn(f"argument {arg} is not a valid parameter (ignored)")
-			return ':' in arg
-
-		def insert(params, path, value):
-			if len(path) == 1:
-				params[path[0]] = value
-				return
-			if not path[0] in params:
-				params[path[0]] = {}
-			insert(params[path[0]], path[1:], value)
-
-		args = [ arg.split(':', 1) for arg in args if check(arg) ]
-		params = {}
-		for (name, value) in args:
-			insert(params, name.split("."), value)
 
 		if "type" in params:
 			pass
-		elif(len(mode) == 1):
+		elif len(mode) == 1:
 			params["type"] = { "b": "binary", "i": "image", "f": "font" }[mode]
 		else:
 			params["type"] = mode
@@ -158,19 +177,17 @@ def main():
 			warn("type 'image' is deprecated, use 'bopti-image' instead")
 			params["type"] = "bopti-image"
 
-		# Use the custom module
+	# Use the custom module
-		custom = None
+	custom = None
-		if use_custom:
+	if use_custom:
-			if converters is None:
+		if converters is None:
-				err("--custom specified but no [converters] module in wd")
+			return err("--custom specified but no [converters] module")
-				sys.exit(1)
+		custom = converters.convert
-			custom = converters.convert
 
-		try:
+	fxconv.convert(input, params, target, output, model, custom)
-			fxconv.convert(input, params, target, output, model, custom)
-		except fxconv.FxconvError as e:
-			err(e)
-			sys.exit(1)
 
 if __name__ == "__main__":
-	main()
+	try:
+		sys.exit(main())
+	except fxconv.FxconvError as e:
+		sys.exit(err(e))

fxconv/fxconv.py

@@ -162,7 +162,7 @@ def ref(base, offset=None, padding=None):
 		assert padding is None
 		return Ref(b"", base, offset or 0)
 
-	raise FxconvException(f"invalid type {type(base)} for ref()")
+	raise FxconvError(f"invalid type {type(base)} for ref()")
 
 Ref = collections.namedtuple("Ref", ["data", "name", "offset"])