diff --git a/reference/bin2c.py b/reference/bin2c.py new file mode 100644 index 0000000..336b4ee --- /dev/null +++ b/reference/bin2c.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python + +from argparse import ArgumentParser +import os + +def main(): + parser = ArgumentParser() + parser.add_argument('src_bin', metavar='SRC', help='source binary data') + parser.add_argument('dst_out', metavar='DST', help='destination c source') + args = parser.parse_args() + + with open(args.src_bin, 'rb') as f: + in_data = f.read() + + transtab = str.maketrans('-.', '__') + varname = os.path.basename(args.src_bin).translate(transtab) + + out_data = '' + + data_len = len(in_data) + n = 0 + while n < data_len: + out_data += ' ' + for i in range(12): + out_data += '0x%02X' % in_data[n] + n += 1 + if n == data_len: + break + elif i == 11: + out_data += ',' + else: + out_data += ', ' + + out_data += '\n' + if n >= data_len: + break + + source_code = \ +f'''#include +#include + +const size_t {varname}_len = {data_len}; +const __attribute__((aligned(4))) uint8_t {varname}[] = {{ +{out_data}}}; +''' + + with open(args.dst_out, 'w') as f: + f.write(source_code) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/reference/mkfrogfs.py b/reference/mkfrogfs.py new file mode 100644 index 0000000..5045092 --- /dev/null +++ b/reference/mkfrogfs.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python + +import configparser +import csv +import gzip +import os +import sys +from argparse import ArgumentParser +from struct import Struct +from zlib import crc32 + +import heatshrink2 + +frogfs_fs_header_t = Struct('= initial_data_len: + flags &= ~FROGFS_FLAG_GZIP + compression = FROGFS_COMPRESSION_NONE + data = inital_data + data_len = initial_data_len + + if initial_data_len < 1024: + initial_data_len_str = f'{initial_data_len:d} B' + data_len_str = f'{data_len:d} B' + elif initial_data_len < 1024 * 1024: + initial_data_len_str = f'{initial_data_len / 1024:.1f} KiB' + data_len_str = f'{data_len / 1024:.1f} KiB' + else: + initial_data_len_str = f'{initial_data_len / 1024 / 1024:.1f} MiB' + data_len_str = f'{data_len / 1024 / 1024:.1f} MiB' + + percent = 100.0 + if initial_data_len > 0: + percent *= data_len / initial_data_len + + stats = f'{initial_data_len_str:<9s} -> {data_len_str:<9s} ({percent:.1f}%)' + print(f'{item[0][0]:08x} {item[0][1]:<34s} file {stats}') + + if flags & FROGFS_FLAG_GZIP: + initial_data_len = data_len + + path = item[0][1].encode('utf8') + b'\0' + path = path.ljust((len(path) + 3) // 4 * 4, b'\0') + header = frogfs_object_header_t.pack(FROGFS_TYPE_FILE, + frogfs_object_header_t.size + frogfs_file_header_t.size, + item[1]['index'], len(path), 0) + frogfs_file_header_t.pack( + data_len, initial_data_len, flags, compression, 0) + + return header + path + data + +def main(): + global args, config + + parser = ArgumentParser() + parser.add_argument('src_dir', metavar='SRC', help='source directory') + parser.add_argument('dst_bin', metavar='DST', help='destination binary') + parser.add_argument('--config', help='user configuration') + args = parser.parse_args() + + config = configparser.ConfigParser() + config.read(os.path.join(args.src_dir, '.config')) + + state = load_state(args.src_dir) + + num_objects = len(state) + offset = frogfs_fs_header_t.size + \ + (frogfs_hashtable_entry_t.size * num_objects) + \ + (frogfs_sorttable_entry_t.size * num_objects) + hashtable = b'' + sorttable = bytearray(frogfs_sorttable_entry_t.size * num_objects) + objects = b'' + + for item in state.items(): + abspath = os.path.join(args.src_dir, item[0][1]) + if item[1]['type'] == 'dir': + object = make_dir_object(item) + elif item[1]['type'] == 'file': + if not os.path.exists(abspath): + continue + with open(abspath, 'rb') as f: + data = f.read() + object = make_file_object(item, data) + else: + print(f'unknown object type {type}', file=sys.stderr) + sys.exit(1) + hashtable += frogfs_hashtable_entry_t.pack(item[0][0], offset) + frogfs_sorttable_entry_t.pack_into(sorttable, + frogfs_sorttable_entry_t.size * item[1]['index'], offset) + objects += object + offset += len(object) + + binary_len = offset + frogfs_crc32_footer_t.size + header = frogfs_fs_header_t.pack(FROGFS_MAGIC, frogfs_fs_header_t.size, + FROGFS_VERSION_MAJOR, FROGFS_VERSION_MINOR, binary_len, num_objects, + 0) + binary = header + hashtable + sorttable + objects + binary += frogfs_crc32_footer_t.pack(crc32(binary) & 0xFFFFFFFF) + + with open(args.dst_bin, 'wb') as f: + f.write(binary) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/reference/preprocess.py b/reference/preprocess.py new file mode 100644 index 0000000..52a8257 --- /dev/null +++ b/reference/preprocess.py @@ -0,0 +1,335 @@ +#!/usr/bin/env python + +import csv +from configparser import ConfigParser +import json +import os +import shutil +import subprocess +import sys +from argparse import ArgumentParser +from collections import OrderedDict +from fnmatch import fnmatch + +script_dir = os.path.dirname(os.path.realpath(__file__)) +used_preprocessors = set() + +def load_config(user_config_file=None): + global config + + defaults_file = os.path.join(script_dir, '..', 'frogfs_defaults.json') + with open(defaults_file) as f: + config = json.load(f) + + user_config = OrderedDict() + if user_config_file: + if not os.path.exists(user_config_file): + print('{user_config_file} cannot be opened', file=sys.stderr) + sys.exit(1) + with open(user_config_file) as f: + user_config = json.load(f) + else: + print("Not loading user config file") + + def merge_section(sec_name): + if sec_name in user_config: + for subsec_name, subsec in user_config[sec_name].items(): + if subsec is None: + if subsec_name in config[sec_name]: + del config[sec_name][subsec_name] + else: + if sec_name == 'filters': + if subsec_name not in config[sec_name]: + config[sec_name] = [] + if isinstance(config[sec_name][subsec_name], str): + config[sec_name][subsec_name] = \ + [config[sec_name][subsec_name]] + if isinstance(subsec, str): + subsec = [subsec] + config[sec_name][subsec_name] += subsec + else: + config[sec_name][subsec_name] = subsec + + for sec_name in ('preprocessors', 'compressors', 'filters'): + merge_section(sec_name) + for subsec_name, subsec in config.get(sec_name, OrderedDict()).items(): + if isinstance(subsec, str): + config[sec_name][subsec_name] = [subsec] + elif isinstance(subsec, dict): + for subsubsec_name, subsubsec in subsec.items(): + if isinstance(subsubsec, str): + subsec[subsubsec_name] = [subsubsec] + + class pattern_sort: + def __init__(self, path, *args): + self.pattern, _ = path + + def __lt__(self, other): + if self.pattern == '*': + return False + if other.pattern == '*': + return True + if self.pattern.startswith('*') and \ + not other.pattern.startswith('*'): + return False + if not self.pattern.startswith('*') and \ + other.pattern.startswith('*'): + return True + return self.pattern < other.pattern + + config['filters'] = OrderedDict(sorted(config['filters'].items(), + key = pattern_sort)) + + preprocessors = list(config['preprocessors'].keys()) + actions = list() + for action in preprocessors + ['cache', 'discard']: + actions.append(action) + actions.append('no-' + action) + actions += ['skip-preprocessing', 'gzip', 'heatshrink', 'uncompressed'] + config['actions'] = actions + + for filter, actions in config['filters'].items(): + for action in actions: + if action not in config['actions']: + print(f"Unknown action `{action}' for filter `{filter}'", + file=sys.stderr) + sys.exit(1) + +def get_preprocessors(path): + global config, used_preprocessors + + preprocessors = OrderedDict() + for pattern, actions in config['filters'].items(): + if fnmatch(path, pattern): + for action in actions: + enable = not action.startswith('no-') + if not enable: + action = action[3:] + if action in config['preprocessors']: + if enable: + preprocessors[action] = None + used_preprocessors.add(action) + else: + try: + del preprocessors[action] + except: + pass + preprocessors[action] = enable + if 'skip-preprocessing' in actions: + return () + + return tuple(preprocessors) + +def get_flags(path): + global config + + flags = OrderedDict() + for pattern, actions in config['filters'].items(): + if fnmatch(path, pattern): + for action in actions: + enable = not action.startswith('no-') + if not enable: + action = action[3:] + if action in ('cache', 'discard', 'skip'): + flags[action] = enable + + return flags + +def get_compressor(path): + global config + + compressor = 'uncompressed' + for pattern, actions in config['filters'].items(): + if fnmatch(path, pattern): + for action in actions: + if action in ('gzip', 'heatshrink', 'uncompressed'): + compressor = action + return compressor + +def load_state(dst_dir): + state = dict() + state_file = os.path.join(dst_dir, '.state') + if os.path.exists(state_file): + with open(state_file, newline='') as f: + reader = csv.reader(f, quoting=csv.QUOTE_NONNUMERIC) + for data in reader: + path, type, mtime, flags, preprocessors, compressor = data + state[path] = { + 'type': type, + 'mtime': mtime, + 'preprocessors': () if not preprocessors else \ + tuple(preprocessors.split(',')), + 'flags': () if not flags else tuple(flags.split(',')), + 'compressor': compressor, + } + return state + +def save_state(dst_dir, state): + with open(os.path.join(dst_dir, '.state'), 'w', newline='') as f: + writer = csv.writer(f, quoting=csv.QUOTE_NONNUMERIC) + for path, data in state.items(): + row = (path, data['type'], data['mtime'], + ','.join(data['flags']), + ','.join(data['preprocessors']), + data['compressor']) + writer.writerow(row) + + dotconfig = ConfigParser() + dotconfig['gzip'] = { + 'level': config['compressors']['gzip']['level'], + } + dotconfig['heatshrink'] = { + 'window_sz2': config['compressors']['heatshrink']['window_sz2'], + 'lookahead_sz2': config['compressors']['heatshrink']['lookahead_sz2'], + } + with open(os.path.join(dst_dir, '.config'), 'w') as f: + dotconfig.write(f) + +def build_state(src_dir): + state = dict() + for dir, _, files in os.walk(src_dir, followlinks=True): + reldir = os.path.relpath(dir, src_dir).replace('\\', '/').lstrip('.') \ + .lstrip('/') + absdir = os.path.abspath(dir) + if reldir and os.path.exists(absdir): + state[reldir] = { + 'type': 'dir', + 'mtime': os.path.getmtime(absdir), + 'preprocessors': (), + 'flags': (), + 'compressor': 'uncompressed', + } + for file in files: + relfile = os.path.join(reldir, file).replace('\\','/').lstrip('/') + absfile = os.path.join(absdir, file) + if os.path.exists(absfile): + state[relfile] = { + 'type': 'file', + 'mtime': os.path.getmtime(absfile), + 'preprocessors': get_preprocessors(relfile), + 'flags': get_flags(relfile), + 'compressor': get_compressor(relfile), + } + return state + +def install_preprocessors(config, root_dir): + global used_preprocessors + + # Work around a bug in npm -- if `node_modules` doesn't exist it will + # create one at some random path above. Sometimes. Depending on the version. + # Except on Tuesdays in Norway. + node_modules = os.path.join(root_dir.replace('/', os.path.sep), 'node_modules') + if not os.path.exists(node_modules): + os.mkdir(node_modules) + + for name in used_preprocessors: + preprocessor = config['preprocessors'][name] + if 'install' in preprocessor: + install = preprocessor['install'] + subprocess.check_call(install, shell=True, cwd=root_dir) + elif 'npm' in preprocessor: + for npm in preprocessor['npm']: + test_path = os.path.join(root_dir.replace('/', os.path.sep), + 'node_modules', + npm.replace('/', os.path.sep)) + if not os.path.exists(test_path): + subprocess.check_call(f'npm install {npm}', shell=True, cwd=root_dir) + +def preprocess(path, preprocessors): + global config + + src_abs = os.path.join(args.src_dir, path) + dst_abs = os.path.join(args.dst_dir, path) + if os.path.isdir(src_abs): + if os.path.isdir(dst_abs): + pass + else: + os.mkdir(dst_abs) + else: + os.makedirs(os.path.dirname(dst_abs), exist_ok=True) + + with open(src_abs, 'rb') as f: + data = f.read() + + if preprocessors: + print(f' - preprocessing {path}', file=sys.stderr) + + for preprocessor in preprocessors: + print(f' - running {preprocessor}') + command = config['preprocessors'][preprocessor]['command'] + if command[0].startswith('tools/'): + command[0] = os.path.join(script_dir, command[0][6:]) + # These are implemented as `.cmd` files on Windows, which explicitly + # requires them to be run under `cmd /c` + if os.name == 'nt': + command = ["cmd", "/c"] + command + process = subprocess.Popen(command, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, shell=True) + data = process.communicate(input=data)[0] + + with open(dst_abs, 'wb') as f: + f.write(data) + +def main(): + global args, config + + parser = ArgumentParser() + parser.add_argument('src_dir', metavar='SRC', help='source directory') + parser.add_argument('dst_dir', metavar='DST', help='destination directory') + parser.add_argument('--config', help='user configuration') + parser.add_argument('--root', metavar='ROOT', help='build root directory') + args = parser.parse_args() + + load_config(args.config) + + old_state = load_state(args.dst_dir) + new_state = build_state(args.src_dir) + + print(f"Root: {args.root}") + install_preprocessors(config, args.root) + + old_paths = set(old_state.keys()) + new_paths = set(new_state.keys()) + + delete_paths = old_paths - new_paths + copy_paths = new_paths - old_paths + compare_paths = old_paths & new_paths + + if not delete_paths and not copy_paths and not compare_paths: + sys.exit(0) + + for path in delete_paths: + dst_abs = os.path.join(args.dst_dir, path) + if os.path.exists(dst_abs): + if os.path.isdir(dst_abs): + shutil.rmtree(dst_abs, True) + else: + os.unlink(dst_abs) + + for path in copy_paths: + preprocess(path, new_state[path]['preprocessors']) + + changes = bool(delete_paths or copy_paths) + for path in compare_paths: + if old_state[path]['type'] != new_state[path]['type'] or \ + old_state[path]['preprocessors'] != \ + new_state[path]['preprocessors'] or \ + old_state[path]['mtime'] < new_state[path]['mtime']: + + changes = True + + dst_abs = os.path.join(args.dst_dir, path) + + if os.path.exists(dst_abs): + if os.path.isdir(dst_abs): + shutil.rmtree(dst_abs, True) + else: + os.unlink(dst_abs) + + preprocess(path, new_state[path]['preprocessors']) + + if changes: + save_state(args.dst_dir, new_state) + +if __name__ == '__main__': + main() diff --git a/reference/requirements.txt b/reference/requirements.txt new file mode 100644 index 0000000..1df6f7a --- /dev/null +++ b/reference/requirements.txt @@ -0,0 +1 @@ +heatshrink2>=0.12.0 diff --git a/reference/zeroify.py b/reference/zeroify.py new file mode 100644 index 0000000..474e88a --- /dev/null +++ b/reference/zeroify.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python + +import sys + + +if __name__ == '__main__': + data = sys.stdin.buffer.read() + sys.stdout.buffer.write(data + b'\0')