fxconv: code review and color image conversion

This change enhances the style of fxconv by using more classes and
generally more Pythonic constructions.

It also introduces image conversion for fx-CG 50, requiring the use
of --fx or --cg to specify the target machine with -i. The default
is set to --fx to maintain compatibility with older Makefiles.

fxconv/fxconv-main.py

@@ -8,7 +8,7 @@ import fxconv
 help_string = """
 usage: fxconv [-s] <python script> [files...]
        fxconv  -b  <bin file> -o <object file> [parameters...]
-       fxconv  -i  <png file> -o <object file> [parameters...]
+       fxconv  -i  <png file> -o <object file> (--fx|--cg) [parameters...]
        fxconv  -f  <png file> -o <object file> [parameters...]
 
 fxconv converts data files such as images and fonts into gint formats
@@ -27,11 +27,14 @@ a Makefile, used to convert only a subset of the files in the script.
 The -b, -i and -f modes are shortcuts to convert single files without a script.
 They accept parameters with a "category.key:value" syntax, for example:
   fxconv -f myfont.png -o myfont.o charset:ascii grid.padding:1 height:7
+
+When converting images, use --fx (black-and-white calculators) or --cg (16-bit
+color calculators) to specify the target machine.
 """.strip()
 
 # Simple error-warnings system
 def err(msg):
-	print("error:", msg, file=sys.stderr)
+	print("\x1b[31;1merror:\x1b[0m", msg, file=sys.stderr)
 def warn(msg):
 	print("warning:", msg, file=sys.stderr)
 
@@ -40,6 +43,7 @@ def main():
 	modes = "script binary image font"
 	mode = "s"
 	output = None
+	target = None
 
 	# Parse command-line arguments
 
@@ -49,7 +53,7 @@ def main():
 
 	try:
 		opts, args = getopt.gnu_getopt(sys.argv[1:], "hsbifo:",
-			("help output="+modes).split())
+			("help output= fx cg "+modes).split())
 	except getopt.GetoptError as error:
 		err(error)
 		sys.exit(1)
@@ -64,6 +68,8 @@ def main():
 			pass
 		elif name in [ "-o", "--output" ]:
 			output = value
+		elif name in [ "--fx", "--cg" ]:
+			target = name[2:]
 		# Other names are modes
 		else:
 			mode = name[1] if len(name)==2 else name[2]
@@ -107,6 +113,11 @@ def main():
 
 		params["type"] = { "b": "binary", "i": "image", "f": "font" }[mode]
 
-		fxconv.convert(input, params, output)
+		try:
+			fxconv.convert(input, params, output, target)
+		except fxconv.FxconvError as e:
+			err(e)
+			sys.exit(1)
 
-main()
+if __name__ == "__main__":
+	main()

fxconv/fxconv.py

@@ -1,5 +1,5 @@
 """
-fxconv: Convert data files into gint formats or object files
+Convert data files into gint formats or object files
 """
 
 import os
@@ -8,8 +8,15 @@ import subprocess
 
 from PIL import Image
 
+__all__ = [
+	# Color names
+	"FX_BLACK", "FX_DARK", "FX_LIGHT", "FX_WHITE", "FX_ALPHA",
+	# Functions
+	"quantize", "convert", "elf",
+]
+
 #
-# Color quantification
+#  Constants
 #
 
 # Colors
@@ -19,89 +26,173 @@ FX_LIGHT  = (170, 170, 170, 255)
 FX_WHITE  = (255, 255, 255, 255)
 FX_ALPHA  = (  0,   0,   0,   0)
 
-# Profiles
+# fx-9860G profiles
+class FxProfile:
+	def __init__(self, id, name, colors, layers):
+		"""
+		Construct an FxProfile object.
+		* [id] is the profile ID in bopti
+		* [name] is the profile's name as seen in the "profile" key
+		* [colors] is the set of supported colors
+		* [layers] is a list of layer functions
+		"""
+
+		self.id = id
+		self.name = name
+		self.gray = FX_LIGHT in colors or FX_DARK in colors
+		self.colors = colors
+		self.layers = layers
+
+	@staticmethod
+	def find(name):
+		"""Find a profile by name."""
+		for profile in FX_PROFILES:
+			if profile.name == name:
+				return profile
+		return None
 
 FX_PROFILES = [
-	{ # Usual black-and-white bitmaps without transparency, as in MonochromeLib
+	# Usual black-and-white bitmaps without transparency, as in MonochromeLib
-	  "name":   "mono",
+	FxProfile(0x0, "mono", { FX_BLACK, FX_WHITE }, [
-	  "gray":   False,
+		lambda c: (c == FX_BLACK),
-	  "colors": { FX_BLACK, FX_WHITE },
+	]),
-	  "layers": [ lambda c: (c == FX_BLACK) ]
+    # Black-and-white with transparency, equivalent of two bitmaps in ML
-	},
+	FxProfile(0x1, "mono_alpha", { FX_BLACK, FX_WHITE, FX_ALPHA }, [
-    { # Black-and-white with transparency, equivalent of two bitmaps in ML
+		lambda c: (c != FX_ALPHA),
-	  "name":   "mono_alpha",
+		lambda c: (c == FX_BLACK),
-	  "gray":   False,
+	]),
-	  "colors": { FX_BLACK, FX_WHITE, FX_ALPHA },
+	# Gray engine bitmaps, reference could have been Eiyeron's Gray Lib
-	  "layers": [ lambda c: (c != FX_ALPHA),
+	FxProfile(0x2, "gray", { FX_BLACK, FX_DARK, FX_LIGHT, FX_WHITE }, [
-	              lambda c: (c == FX_BLACK) ]
+		lambda c: (c in [FX_BLACK, FX_LIGHT]),
-	},
+		lambda c: (c in [FX_BLACK, FX_DARK]),
-	{ # Gray engine bitmaps, reference could have been Eiyeron's Gray Lib
+	]),
-	  "name":   "gray",
+	# Gray images with transparency, unfortunately 3 layers since 5 colors
-	  "gray":   True,
+	FxProfile(0x3, "gray_alpha",
-	  "colors": { FX_BLACK, FX_DARK, FX_LIGHT, FX_WHITE },
+		{ FX_BLACK, FX_DARK, FX_LIGHT, FX_WHITE, FX_ALPHA }, [
-	  "layers": [ lambda c: (c in [FX_BLACK, FX_LIGHT]),
+		lambda c: (c != FX_ALPHA),
-	              lambda c: (c in [FX_BLACK, FX_DARK]) ]
+		lambda c: (c in [FX_BLACK, FX_LIGHT]),
-	},
+		lambda c: (c in [FX_BLACK, FX_DARK]),
-	{ # Gray images with transparency, unfortunately 3 layers since 5 colors
+	]),
-	  "name":   "gray_alpha",
+]
-	  "gray":   True,
+
-	  "colors": { FX_BLACK, FX_DARK, FX_LIGHT, FX_WHITE, FX_ALPHA },
+# fx-CG 50 profiles
-	  "layers": [ lambda c: (c != FX_ALPHA),
+class CgProfile:
-	              lambda c: (c in [FX_BLACK, FX_LIGHT]),
+	def __init__(self, id, name, alpha):
-	              lambda c: (c in [FX_BLACK, FX_DARK]) ]
+		"""
-	},
+		Construct a CgProfile object.
+		* [id] is the profile ID in bopti
+		* [name] is the profile name as found in the specification key
+		* [alpha] is True if this profile supports alpha, False otherwise
+		"""
+
+		self.id = id
+		self.name = name
+		self.supports_alpha = alpha
+
+	@staticmethod
+	def find(name):
+		"""Find a profile by name."""
+		for profile in CG_PROFILES:
+			if profile.name == name:
+				return profile
+		return None
+
+CG_PROFILES = [
+	# 16-bit R5G6B5
+	CgProfile(0x0, "r5g6b5", False),
+	# 16-bit R5G6B5 with alpha
+	CgProfile(0x1, "r5g6b5a", True),
+	# 8-bit palette
+	CgProfile(0x2, "p8", True),
+	# 4-bit palette
+	CgProfile(0x3, "p4", True),
 ]
 
 #
-# Character sets
+#  Character sets
 #
 
-class _Charset:
+class Charset:
 	def __init__(self, id, name, count):
 		self.id = id
 		self.name = name
 		self.count = count
 
+	@staticmethod
+	def find(name):
+		"""Find a charset by name."""
+		for charset in FX_CHARSETS:
+			if charset.name == name:
+				return charset
+		return None
+
 FX_CHARSETS = [
 	# Digits 0...9
-	_Charset(0x0, "numeric", 10),
+	Charset(0x0, "numeric", 10),
 	# Uppercase letters A...Z
-	_Charset(0x1, "upper", 26),
+	Charset(0x1, "upper", 26),
 	# Upper and lowercase letters A..Z, a..z
-	_Charset(0x2, "alpha", 52),
+	Charset(0x2, "alpha", 52),
 	# Letters and digits A..Z, a..z, 0..9
-	_Charset(0x3, "alnum", 62),
+	Charset(0x3, "alnum", 62),
 	# All printable characters from 0x20 to 0x7e
-	_Charset(0x4, "print", 95),
+	Charset(0x4, "print", 95),
 	# All 128 ASII characters
-	_Charset(0x5, "ascii", 128),
+	Charset(0x5, "ascii", 128),
 ]
 
 #
-# Internal routines
+#  Area specifications
 #
 
-# normalize_area(): Expand area.size and set defaults for all values.
+class Area:
-def _normalize_area(area, img):
+	def __init__(self, area, img):
-	default = { "x": 0, "y": 0, "width": img.width, "height": img.height }
+		"""
-	if area is None:
+		Construct an Area object from a dict specification. The following keys
-		area = default
+		may be used:
-	else:
+
+		* "x", "y"          (int strings, default to 0)
+		* "width", "height" (int strings, default to image dimensions)
+		* "size"            ("WxH" where W and H are the width and height)
+
+		The Area objects has attributes "x", "y", "w" and "h".
+		"""
+
+		self.x = int(area.get("x", 0))
+		self.y = int(area.get("y", 0))
+		self.w = int(area.get("width", img.width))
+		self.h = int(area.get("height", img.height))
+
 		if "size" in area:
-			area["width"], area["height"] = area["size"].split("x")
+			self.w, self.h = map(int, area["size"].split("x"))
-		area = { **default, **area }
 
-	return (int(area[key]) for key in "x y width height".split())
+	def tuple(self):
+		"""Return the tuple representation (x,y,w,h)."""
+		return (self.x, self.y, self.w, self.h)
 
-class _Grid:
+#
-	# [grid] is a dictionary of parameters. Relevant keys:
+#  Grid specifications
-	#   "border", "padding", "width", "height", "size"
+#
+
+class Grid:
 	def __init__(self, grid):
-		self.border  = int(grid.get("border",  0))
+		"""
+		Construct a Grid object from a dict specification. The following keys
+		may be used:
+
+		* "border"           (int string, defaults to 0)
+		* "padding"          (int string, defaults to 0)
+		* "width", "height"  (int strings, mandatory if "size" not set)
+		* "size"             ("WxH" where W and H are the cell width/height)
+
+		The Grid object has attributes "border", "padding", "w" and "h".
+		"""
+
+		self.border = int(grid.get("border", 0))
 		self.padding = int(grid.get("padding", 0))
 
-		self.w = int(grid.get("width", "-1"))
+		self.w = int(grid.get("width", -1))
-		self.h = int(grid.get("height", "-1"))
+		self.h = int(grid.get("height", -1))
 
 		if "size" in grid:
 			self.w, self.h = map(int, grid["size"].split("x"))
@@ -109,8 +200,8 @@ class _Grid:
 		if self.w <= 0 or self.h <= 0:
 			raise FxconvError("size of grid unspecified or invalid")
 
-	# size(): Number of elements in the grid
 	def size(self, img):
+		"""Count the number of elements in the grid."""
 		b, p, w, h =  self.border, self.padding, self.w, self.h
 
 		# Padding-extended parameters
@@ -122,8 +213,8 @@ class _Grid:
 		return columns * rows
 
 
-	# iter(): Iterator on all rectangles of the grid
 	def iter(self, img):
+		"""Build an iterator on all subrectangles of the grid."""
 		b, p, w, h =  self.border, self.padding, self.w, self.h
 
 		# Padding-extended parameters
@@ -143,25 +234,22 @@ class _Grid:
 # Binary conversion
 #
 
-def _convert_binary(input, output, params):
+def convert_binary(input, output, params):
-	raise FxconvError("TODO: binary mode x_x")
+	data = open(input, "rb").read()
+	elf(data, output, "_" + params["name"])
 
 #
-# Image conversion
+# Image conversion for fx-9860G
 #
 
-def _profile_find(name):
+def convert_bopti_fx(input, output, params):
-	gen = ((i,pr) for (i,pr) in enumerate(FX_PROFILES) if pr["name"] == name)
-	return next(gen, (None,None))
-
-def _convert_image(input, output, params):
 	img = Image.open(input)
 	if img.width >= 4096 or img.height >= 4096:
-		raise FxconvError(f"'{input}' is too large (max. 4095*4095)")
+		raise FxconvError(f"'{input}' is too large (max. 4095x4095)")
 
 	# Expand area.size and get the defaults. Crop image to resulting area.
-	params["area"] = _normalize_area(params.get("area", None), img)
+	area = Area(params.get("area", {}), img)
-	img = img.crop(params["area"])
+	img = img.crop(area.tuple())
 
 	# Quantize the image and check the profile
 	img = quantize(img, dither=False)
@@ -172,26 +260,27 @@ def _convert_image(input, output, params):
 	colors = { y for (x,y) in img.getcolors() }
 
 	if "profile" in params:
-		p = params["profile"]
+		name = params["profile"]
-		pid, p = _profile_find(p)
+		p = FxProfile.find(name)
+
 		if p is None:
-			raise FxconvError(f"unknown profile {p} in conversion '{input}'")
+			raise FxconvError(f"unknown profile {name} in '{input}'")
-		if colors - profiles[p]:
+		if colors - p.colors:
-			raise FxconvError(f"'{input}' has more colors than profile '{p}'")
+			raise FxconvError(f"{name} has too few colors for '{input}'")
 	else:
-		p = "gray" if FX_LIGHT in colors or FX_DARK in colors else "mono"
+		name = "gray" if FX_LIGHT in colors or FX_DARK in colors else "mono"
-		if FX_ALPHA in colors: p += "_alpha"
+		if FX_ALPHA in colors: name += "_alpha"
-		pid, p = _profile_find(p)
+		p = FxProfile.find(name)
 
 	# Make the image header
 
-	header = bytes ([(0x80 if p["gray"] else 0) + pid])
+	header = bytes ([(0x80 if p.gray else 0) + p.id])
 	encode24bit = lambda x: bytes([ x >> 16, (x & 0xff00) >> 8, x & 0xff ])
 	header += encode24bit((img.size[0] << 12) + img.size[1])
 
 	# Split the image into layers depending on the profile and zip them all
 
-	layers = [ _image_project(img, layer) for layer in p["layers"] ]
+	layers = [ _image_project(img, layer) for layer in p.layers ]
 	count  = len(layers)
 	size   = len(layers[0])
 
@@ -224,25 +313,57 @@ def _image_project(img, f):
 
 	return data
 
+#
+# Image conversion for fx-CG 50
+#
+
+def convert_bopti_cg(input, output, params):
+	img = Image.open(input)
+	if img.width >= 65536 or img.height >= 65536:
+		raise FxconvError(f"'{input}' is too large (max. 65535x65535)")
+
+	# Crop image to key "area"
+	area = Area(params.get("area", {}), img)
+	img = img.crop(area.tuple())
+
+	# Encode the image into the 16-bit format
+	encoded, alpha = r5g6b5(img)
+
+	# If no profile is specified, fall back to R5G6B5 or R5G6B5A as needed
+	name = params.get("profile", "r5g6b5" if alpha is None else "r5g6b5a")
+	profile = CgProfile.find(name)
+
+	if name in [ "r5g6b5", "r5g6b5a" ]:
+
+		if alpha is not None and not profile.supports_alpha:
+			raise FxconvError(f"'{input}' has transparency; use r5g6b5a")
+
+		w, h, a = img.width, img.height, (0x00 if alpha is None else alpha)
+
+		header = bytearray([
+			0x00, profile.id,		# Profile identification
+			a >> 8, a & 0xff,		# Alpha color
+			w >> 8, w & 0xff,		# Width
+			h >> 8, h & 0xff,		# Height
+		])
+
+		elf(header + encoded, output, "_" + params["name"])
+
 #
 # Font conversion
 #
 
-def _charset_find(name):
-	gen = (cs for cs in FX_CHARSETS if cs.name == name)
-	return next(gen, None)
-
 def _trim(img):
-	def _blank(x):
+	def blank(x):
 		return all(px[x,y] == FX_WHITE for y in range(img.height))
 
 	left = 0
 	right = img.width
 	px = img.load()
 
-	while left + 1 < right and _blank(left):
+	while left + 1 < right and blank(left):
 		left += 1
-	while right - 1 > left and _blank(right - 1):
+	while right - 1 > left and blank(right - 1):
 		right -= 1
 
 	return img.crop((left, 0, right, img.height))
@@ -255,19 +376,21 @@ def _pad(seq, length):
 	n = max(0, length - len(seq))
 	return seq + bytearray(n)
 
-def _convert_font(input, output, params):
+def convert_topti(input, output, params):
 
 	#--
 	# Image area and grid
 	#--
 
 	img = Image.open(input)
-	params["area"] = _normalize_area(params.get("area", None), img)
+	area = Area(params.get("area", {}), img)
-	img = img.crop(params["area"])
+	img = img.crop(area.tuple())
 
-	grid = _Grid(params.get("grid", {}))
+	grid = Grid(params.get("grid", {}))
 
-	# Quantize image (any profile will do)
+	# Quantize image. (Profile doesn't matter here; only black pixels will be
+	# encoded into glyphs. White pixels are used to separate entries and gray
+	# pixels can be used to forcefully insert spacing on the sides.)
 	img = quantize(img, dither=False)
 
 	#--
@@ -277,7 +400,7 @@ def _convert_font(input, output, params):
 	if "charset" not in params:
 		raise FxconvError("'charset' attribute is required and missing")
 
-	charset = _charset_find(params["charset"])
+	charset = Charset.find(params["charset"])
 	if charset is None:
 		raise FxconvError(f"unknown character set '{charset}'")
 	if charset.count > grid.size(img):
@@ -371,7 +494,8 @@ def _convert_font(input, output, params):
 # Exceptions
 #
 
-FxconvError = Exception
+class FxconvError(Exception):
+	pass
 
 #
 # API
@@ -379,7 +503,7 @@ FxconvError = Exception
 
 def quantize(img, dither=False):
 	"""
-	Convert a PIL.Image.Image into an RGBA image whose only colors are:
+	Convert a PIL.Image.Image into an RGBA image with only these colors:
 	* FX_BLACK  = (  0,   0,   0, 255)
 	* FX_DARK   = ( 85,  85,  85, 255)
 	* FX_LIGHT  = (170, 170, 170, 255)
@@ -438,7 +562,81 @@ def quantize(img, dither=False):
 
 	return img
 
-def convert(input, params, output=None):
+def r5g6b5(img):
+	"""
+	Convert a PIL.Image.Image into an R5G6B5 byte stream. If there are
+	transparent pixels, chooses a color to implement alpha and replaces them
+	with this color.
+
+	Returns the converted image as a bytearray and the alpha value, or None if
+	no alpha value was used.
+	"""
+
+	def rgb24to16(r, g, b):
+		r = (r & 0xff) >> 3
+		g = (g & 0xff) >> 2
+		b = (b & 0xff) >> 3
+		return (r << 11) | (g << 5) | b
+
+	# Save the alpha channel and make it 1-bit
+	try:
+		alpha_channel = img.getchannel("A").convert("1", dither=Image.NONE)
+	except:
+		alpha_channel = Image.new("L", img.size, 255)
+
+	# Convert the input image to RGB and put back the alpha channel
+	img = img.convert("RGB")
+	img.putalpha(alpha_channel)
+
+	# Gather a list of R5G6B5 colors
+
+	colors = set()
+	has_alpha = False
+
+	pixels = img.load()
+	for y in range(img.height):
+		for x in range(img.width):
+			r, g, b, a = pixels[x, y]
+
+			if a == 0:
+				has_alpha = True
+			else:
+				colors.add(rgb24to16(r, g, b))
+
+	# Choose a color for the alpha if needed
+
+	if has_alpha:
+		palette = set(range(65536))
+		available = palette - colors
+
+		if not available:
+			raise FxconvError("image uses all 65536 colors and alpha")
+		alpha = available.pop()
+	else:
+		alpha = None
+
+	# Create a byte array with all encoded pixels
+
+	encoded = bytearray(img.width * img.height * 2)
+	offset = 0
+
+	for y in range(img.height):
+		for x in range(img.width):
+			r, g, b, a = pixels[x, y]
+
+			if a == 0:
+				encoded[offset]   = alpha >> 8
+				encoded[offset+1] = alpha & 0xff
+			else:
+				rgb16 = rgb24to16(r, g, b)
+				encoded[offset]   = rgb16 >> 8
+				encoded[offset+1] = rgb16 & 0xff
+
+			offset += 2
+
+	return encoded, alpha
+
+def convert(input, params, output=None, target=None):
 	"""
 	Convert a data file into an object that exports the following symbols:
 	* _<varname>
@@ -450,12 +648,13 @@ def convert(input, params, output=None):
 	input   -- Input file path
 	params  -- Parameter dictionary
 	output  -- Output file name [default: <input> with suffix '.o']
+	target  -- 'fx' or 'cg'  (some conversions require this) [default: None]
 
 	Produces an output file and returns nothing.
 	"""
 
 	if output is None:
-		output = os.path.splitext(input)[0] + '.o'
+		output = os.path.splitext(input)[0] + ".o"
 
 	if "name" not in params:
 		raise FxconvError(f"no name specified for conversion '{input}'")
@@ -463,11 +662,13 @@ def convert(input, params, output=None):
 	if "type" not in params:
 		raise FxconvError(f"missing type in conversion '{input}'")
 	elif params["type"] == "binary":
-		_convert_binary(input, output, params)
+		convert_binary(input, output, params)
-	elif params["type"] == "image":
+	elif params["type"] == "image" and target in [ "fx", None ]:
-		_convert_image(input, output, params)
+		convert_bopti_fx(input, output, params)
+	elif params["type"] == "image" and target == "cg":
+		convert_bopti_cg(input, output, params)
 	elif params["type"] == "font":
-		_convert_font(input, output, params)
+		convert_topti(input, output, params)
 
 def elf(data, output, symbol, section=None, arch="sh3"):
 	"""

fxsdk/assets/Makefile

@@ -38,9 +38,9 @@ target-fx := $(filename).g1a
 target-cg := $(filename).g3a
 
 # Source files
-src       := $(shell find src -name '*.c')
+src       := $(wildcard src/*.c src/*/*.c src/*/*/*.c src/*/*/*/*.c)
-assets-fx := $(shell find assets-fx/*/)
+assets-fx := $(wildcard assets-fx/*/*)
-assets-cg := $(shell find assets-cg/*/)
+assets-cg := $(wildcard assets-cg/*/*)
 
 # Object files
 obj-fx  := $(src:%.c=build-fx/%.o) $(assets-fx:assets-fx/%=build-fx/assets/%.o)
@@ -91,13 +91,11 @@ build-cg/%.o: %.c
 # Images
 build-fx/assets/img/%.o: assets-fx/img/%
 	@ mkdir -p $(dir $@)
-	fxconv -i $< -o $@ name:img_$(basename $*)
+	fxconv -i $< -o $@ --fx name:img_$(basename $*)
 
 build-cg/assets/img/%.o: assets-cg/img/%
-	@ echo -ne "\e[31;1mWARNING: image conversion for fxcg50 is not "
-	@ echo -ne "supported yet\e[0m"
 	@ mkdir -p $(dir $@)
-	fxconv -i $< -o $@ name:img_$(basename $*)
+	fxconv -i $< -o $@ --cg name:img_$(basename $*)
 
 # Fonts
 build-fx/assets/fonts/%.o: assets-fx/fonts/%
@@ -126,8 +124,8 @@ distclean: clean
 install-fx: $(target-fx)
 	p7 send -f $<
 install-cg: $(target-cg)
-	@ while [[ ! -h /dev/Prizm1 ]]; do sleep 1; done
+	@ while [[ ! -h /dev/Prizm1 ]]; do sleep 0.25; done
-	@ mount /dev/Prizm1
+	@ while ! mount /dev/Prizm1; do sleep 0.25; done
 	@ rm -f /mnt/prizm/$<
 	@ cp $< /mnt/prizm
 	@ umount /dev/Prizm1