reference: add reference implementation

Signed-off-by: Sean Cross <sean@xobs.io>
This commit is contained in:
Sean Cross 2023-11-20 18:03:23 +08:00
parent 07dc3f87ed
commit bcab1fe755
5 changed files with 589 additions and 0 deletions

51
reference/bin2c.py Normal file
View File

@ -0,0 +1,51 @@
#!/usr/bin/env python
from argparse import ArgumentParser
import os
def main():
parser = ArgumentParser()
parser.add_argument('src_bin', metavar='SRC', help='source binary data')
parser.add_argument('dst_out', metavar='DST', help='destination c source')
args = parser.parse_args()
with open(args.src_bin, 'rb') as f:
in_data = f.read()
transtab = str.maketrans('-.', '__')
varname = os.path.basename(args.src_bin).translate(transtab)
out_data = ''
data_len = len(in_data)
n = 0
while n < data_len:
out_data += ' '
for i in range(12):
out_data += '0x%02X' % in_data[n]
n += 1
if n == data_len:
break
elif i == 11:
out_data += ','
else:
out_data += ', '
out_data += '\n'
if n >= data_len:
break
source_code = \
f'''#include <stddef.h>
#include <stdint.h>
const size_t {varname}_len = {data_len};
const __attribute__((aligned(4))) uint8_t {varname}[] = {{
{out_data}}};
'''
with open(args.dst_out, 'w') as f:
f.write(source_code)
if __name__ == '__main__':
main()

194
reference/mkfrogfs.py Normal file
View File

@ -0,0 +1,194 @@
#!/usr/bin/env python
import configparser
import csv
import gzip
import os
import sys
from argparse import ArgumentParser
from struct import Struct
from zlib import crc32
import heatshrink2
frogfs_fs_header_t = Struct('<IBBHIHH')
# magic, len, version_major, version_minor, binary_len, num_objects, reserved
FROGFS_MAGIC = 0x676F7246 # Frog
FROGFS_VERSION_MAJOR = 1
FROGFS_VERSION_MINOR = 0
frogfs_hashtable_entry_t = Struct('<II')
# hash, offset
frogfs_sorttable_entry_t = Struct('<I')
# offset
frogfs_object_header_t = Struct('<BBHHH')
# type, len, index, path_len, reserved
FROGFS_TYPE_FILE = 0
FROGFS_TYPE_DIR = 1
frogfs_file_header_t = Struct('<IIHBB')
# data_len, file_len, flags, compression, reserved
FROGFS_FLAG_GZIP = (1 << 0)
FROGFS_FLAG_CACHE = (1 << 1)
FROGFS_COMPRESSION_NONE = 0
FROGFS_COMPRESSION_HEATSHRINK = 1
frogfs_heatshrink_header_t = Struct('<BBH')
# window_sz2, lookahead_sz2
frogfs_crc32_footer_t = Struct('<I')
# crc32
def djb2_hash(s):
hash = 5381
for c in s.encode('utf8'):
hash = ((hash << 5) + hash ^ c) & 0xFFFFFFFF
return hash
def load_state(path):
state = dict()
state_file = os.path.join(args.src_dir, '.state')
with open(state_file, newline='') as f:
index = 0
reader = csv.reader(f, quoting=csv.QUOTE_NONNUMERIC)
for data in reader:
path, type, _, flags, _, compressor = data
hash = djb2_hash(path)
flags = () if not flags else tuple(flags.split(','))
if 'discard' in flags:
continue
state[(hash, path)] = {
'index': index,
'type': type,
'flags': flags,
'compressor': compressor,
}
index += 1
return state
def make_dir_object(item):
print(f'{item[0][0]:08x} {item[0][1]:<34s} dir')
path = item[0][1].encode('utf8') + b'\0'
path = path.ljust((len(path) + 3) // 4 * 4, b'\0')
header = frogfs_object_header_t.pack(FROGFS_TYPE_DIR,
frogfs_object_header_t.size, item[1]['index'], len(path), 0)
return header + path
def make_file_object(item, data):
global config
flags = 0
compression = FROGFS_COMPRESSION_NONE
initial_data_len = len(data)
inital_data = data
if 'cache' in item[1]['flags']:
flags |= FROGFS_FLAG_CACHE
if item[1]['compressor'] == 'gzip':
flags |= FROGFS_FLAG_GZIP
level = int(config['gzip']['level'])
level = min(max(level, 0), 9)
data = gzip.compress(data, level)
elif item[1]['compressor'] == 'heatshrink':
compression = FROGFS_COMPRESSION_HEATSHRINK
window_sz2 = int(config['heatshrink']['window_sz2'])
lookahead_sz2 = int(config['heatshrink']['lookahead_sz2'])
data = frogfs_heatshrink_header_t.pack(window_sz2, lookahead_sz2, 0) + \
heatshrink2.compress(data, window_sz2=window_sz2,
lookahead_sz2=lookahead_sz2)
data_len = len(data)
if data_len >= initial_data_len:
flags &= ~FROGFS_FLAG_GZIP
compression = FROGFS_COMPRESSION_NONE
data = inital_data
data_len = initial_data_len
if initial_data_len < 1024:
initial_data_len_str = f'{initial_data_len:d} B'
data_len_str = f'{data_len:d} B'
elif initial_data_len < 1024 * 1024:
initial_data_len_str = f'{initial_data_len / 1024:.1f} KiB'
data_len_str = f'{data_len / 1024:.1f} KiB'
else:
initial_data_len_str = f'{initial_data_len / 1024 / 1024:.1f} MiB'
data_len_str = f'{data_len / 1024 / 1024:.1f} MiB'
percent = 100.0
if initial_data_len > 0:
percent *= data_len / initial_data_len
stats = f'{initial_data_len_str:<9s} -> {data_len_str:<9s} ({percent:.1f}%)'
print(f'{item[0][0]:08x} {item[0][1]:<34s} file {stats}')
if flags & FROGFS_FLAG_GZIP:
initial_data_len = data_len
path = item[0][1].encode('utf8') + b'\0'
path = path.ljust((len(path) + 3) // 4 * 4, b'\0')
header = frogfs_object_header_t.pack(FROGFS_TYPE_FILE,
frogfs_object_header_t.size + frogfs_file_header_t.size,
item[1]['index'], len(path), 0) + frogfs_file_header_t.pack(
data_len, initial_data_len, flags, compression, 0)
return header + path + data
def main():
global args, config
parser = ArgumentParser()
parser.add_argument('src_dir', metavar='SRC', help='source directory')
parser.add_argument('dst_bin', metavar='DST', help='destination binary')
parser.add_argument('--config', help='user configuration')
args = parser.parse_args()
config = configparser.ConfigParser()
config.read(os.path.join(args.src_dir, '.config'))
state = load_state(args.src_dir)
num_objects = len(state)
offset = frogfs_fs_header_t.size + \
(frogfs_hashtable_entry_t.size * num_objects) + \
(frogfs_sorttable_entry_t.size * num_objects)
hashtable = b''
sorttable = bytearray(frogfs_sorttable_entry_t.size * num_objects)
objects = b''
for item in state.items():
abspath = os.path.join(args.src_dir, item[0][1])
if item[1]['type'] == 'dir':
object = make_dir_object(item)
elif item[1]['type'] == 'file':
if not os.path.exists(abspath):
continue
with open(abspath, 'rb') as f:
data = f.read()
object = make_file_object(item, data)
else:
print(f'unknown object type {type}', file=sys.stderr)
sys.exit(1)
hashtable += frogfs_hashtable_entry_t.pack(item[0][0], offset)
frogfs_sorttable_entry_t.pack_into(sorttable,
frogfs_sorttable_entry_t.size * item[1]['index'], offset)
objects += object
offset += len(object)
binary_len = offset + frogfs_crc32_footer_t.size
header = frogfs_fs_header_t.pack(FROGFS_MAGIC, frogfs_fs_header_t.size,
FROGFS_VERSION_MAJOR, FROGFS_VERSION_MINOR, binary_len, num_objects,
0)
binary = header + hashtable + sorttable + objects
binary += frogfs_crc32_footer_t.pack(crc32(binary) & 0xFFFFFFFF)
with open(args.dst_bin, 'wb') as f:
f.write(binary)
if __name__ == '__main__':
main()

335
reference/preprocess.py Normal file
View File

@ -0,0 +1,335 @@
#!/usr/bin/env python
import csv
from configparser import ConfigParser
import json
import os
import shutil
import subprocess
import sys
from argparse import ArgumentParser
from collections import OrderedDict
from fnmatch import fnmatch
script_dir = os.path.dirname(os.path.realpath(__file__))
used_preprocessors = set()
def load_config(user_config_file=None):
global config
defaults_file = os.path.join(script_dir, '..', 'frogfs_defaults.json')
with open(defaults_file) as f:
config = json.load(f)
user_config = OrderedDict()
if user_config_file:
if not os.path.exists(user_config_file):
print('{user_config_file} cannot be opened', file=sys.stderr)
sys.exit(1)
with open(user_config_file) as f:
user_config = json.load(f)
else:
print("Not loading user config file")
def merge_section(sec_name):
if sec_name in user_config:
for subsec_name, subsec in user_config[sec_name].items():
if subsec is None:
if subsec_name in config[sec_name]:
del config[sec_name][subsec_name]
else:
if sec_name == 'filters':
if subsec_name not in config[sec_name]:
config[sec_name] = []
if isinstance(config[sec_name][subsec_name], str):
config[sec_name][subsec_name] = \
[config[sec_name][subsec_name]]
if isinstance(subsec, str):
subsec = [subsec]
config[sec_name][subsec_name] += subsec
else:
config[sec_name][subsec_name] = subsec
for sec_name in ('preprocessors', 'compressors', 'filters'):
merge_section(sec_name)
for subsec_name, subsec in config.get(sec_name, OrderedDict()).items():
if isinstance(subsec, str):
config[sec_name][subsec_name] = [subsec]
elif isinstance(subsec, dict):
for subsubsec_name, subsubsec in subsec.items():
if isinstance(subsubsec, str):
subsec[subsubsec_name] = [subsubsec]
class pattern_sort:
def __init__(self, path, *args):
self.pattern, _ = path
def __lt__(self, other):
if self.pattern == '*':
return False
if other.pattern == '*':
return True
if self.pattern.startswith('*') and \
not other.pattern.startswith('*'):
return False
if not self.pattern.startswith('*') and \
other.pattern.startswith('*'):
return True
return self.pattern < other.pattern
config['filters'] = OrderedDict(sorted(config['filters'].items(),
key = pattern_sort))
preprocessors = list(config['preprocessors'].keys())
actions = list()
for action in preprocessors + ['cache', 'discard']:
actions.append(action)
actions.append('no-' + action)
actions += ['skip-preprocessing', 'gzip', 'heatshrink', 'uncompressed']
config['actions'] = actions
for filter, actions in config['filters'].items():
for action in actions:
if action not in config['actions']:
print(f"Unknown action `{action}' for filter `{filter}'",
file=sys.stderr)
sys.exit(1)
def get_preprocessors(path):
global config, used_preprocessors
preprocessors = OrderedDict()
for pattern, actions in config['filters'].items():
if fnmatch(path, pattern):
for action in actions:
enable = not action.startswith('no-')
if not enable:
action = action[3:]
if action in config['preprocessors']:
if enable:
preprocessors[action] = None
used_preprocessors.add(action)
else:
try:
del preprocessors[action]
except:
pass
preprocessors[action] = enable
if 'skip-preprocessing' in actions:
return ()
return tuple(preprocessors)
def get_flags(path):
global config
flags = OrderedDict()
for pattern, actions in config['filters'].items():
if fnmatch(path, pattern):
for action in actions:
enable = not action.startswith('no-')
if not enable:
action = action[3:]
if action in ('cache', 'discard', 'skip'):
flags[action] = enable
return flags
def get_compressor(path):
global config
compressor = 'uncompressed'
for pattern, actions in config['filters'].items():
if fnmatch(path, pattern):
for action in actions:
if action in ('gzip', 'heatshrink', 'uncompressed'):
compressor = action
return compressor
def load_state(dst_dir):
state = dict()
state_file = os.path.join(dst_dir, '.state')
if os.path.exists(state_file):
with open(state_file, newline='') as f:
reader = csv.reader(f, quoting=csv.QUOTE_NONNUMERIC)
for data in reader:
path, type, mtime, flags, preprocessors, compressor = data
state[path] = {
'type': type,
'mtime': mtime,
'preprocessors': () if not preprocessors else \
tuple(preprocessors.split(',')),
'flags': () if not flags else tuple(flags.split(',')),
'compressor': compressor,
}
return state
def save_state(dst_dir, state):
with open(os.path.join(dst_dir, '.state'), 'w', newline='') as f:
writer = csv.writer(f, quoting=csv.QUOTE_NONNUMERIC)
for path, data in state.items():
row = (path, data['type'], data['mtime'],
','.join(data['flags']),
','.join(data['preprocessors']),
data['compressor'])
writer.writerow(row)
dotconfig = ConfigParser()
dotconfig['gzip'] = {
'level': config['compressors']['gzip']['level'],
}
dotconfig['heatshrink'] = {
'window_sz2': config['compressors']['heatshrink']['window_sz2'],
'lookahead_sz2': config['compressors']['heatshrink']['lookahead_sz2'],
}
with open(os.path.join(dst_dir, '.config'), 'w') as f:
dotconfig.write(f)
def build_state(src_dir):
state = dict()
for dir, _, files in os.walk(src_dir, followlinks=True):
reldir = os.path.relpath(dir, src_dir).replace('\\', '/').lstrip('.') \
.lstrip('/')
absdir = os.path.abspath(dir)
if reldir and os.path.exists(absdir):
state[reldir] = {
'type': 'dir',
'mtime': os.path.getmtime(absdir),
'preprocessors': (),
'flags': (),
'compressor': 'uncompressed',
}
for file in files:
relfile = os.path.join(reldir, file).replace('\\','/').lstrip('/')
absfile = os.path.join(absdir, file)
if os.path.exists(absfile):
state[relfile] = {
'type': 'file',
'mtime': os.path.getmtime(absfile),
'preprocessors': get_preprocessors(relfile),
'flags': get_flags(relfile),
'compressor': get_compressor(relfile),
}
return state
def install_preprocessors(config, root_dir):
global used_preprocessors
# Work around a bug in npm -- if `node_modules` doesn't exist it will
# create one at some random path above. Sometimes. Depending on the version.
# Except on Tuesdays in Norway.
node_modules = os.path.join(root_dir.replace('/', os.path.sep), 'node_modules')
if not os.path.exists(node_modules):
os.mkdir(node_modules)
for name in used_preprocessors:
preprocessor = config['preprocessors'][name]
if 'install' in preprocessor:
install = preprocessor['install']
subprocess.check_call(install, shell=True, cwd=root_dir)
elif 'npm' in preprocessor:
for npm in preprocessor['npm']:
test_path = os.path.join(root_dir.replace('/', os.path.sep),
'node_modules',
npm.replace('/', os.path.sep))
if not os.path.exists(test_path):
subprocess.check_call(f'npm install {npm}', shell=True, cwd=root_dir)
def preprocess(path, preprocessors):
global config
src_abs = os.path.join(args.src_dir, path)
dst_abs = os.path.join(args.dst_dir, path)
if os.path.isdir(src_abs):
if os.path.isdir(dst_abs):
pass
else:
os.mkdir(dst_abs)
else:
os.makedirs(os.path.dirname(dst_abs), exist_ok=True)
with open(src_abs, 'rb') as f:
data = f.read()
if preprocessors:
print(f' - preprocessing {path}', file=sys.stderr)
for preprocessor in preprocessors:
print(f' - running {preprocessor}')
command = config['preprocessors'][preprocessor]['command']
if command[0].startswith('tools/'):
command[0] = os.path.join(script_dir, command[0][6:])
# These are implemented as `.cmd` files on Windows, which explicitly
# requires them to be run under `cmd /c`
if os.name == 'nt':
command = ["cmd", "/c"] + command
process = subprocess.Popen(command, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, shell=True)
data = process.communicate(input=data)[0]
with open(dst_abs, 'wb') as f:
f.write(data)
def main():
global args, config
parser = ArgumentParser()
parser.add_argument('src_dir', metavar='SRC', help='source directory')
parser.add_argument('dst_dir', metavar='DST', help='destination directory')
parser.add_argument('--config', help='user configuration')
parser.add_argument('--root', metavar='ROOT', help='build root directory')
args = parser.parse_args()
load_config(args.config)
old_state = load_state(args.dst_dir)
new_state = build_state(args.src_dir)
print(f"Root: {args.root}")
install_preprocessors(config, args.root)
old_paths = set(old_state.keys())
new_paths = set(new_state.keys())
delete_paths = old_paths - new_paths
copy_paths = new_paths - old_paths
compare_paths = old_paths & new_paths
if not delete_paths and not copy_paths and not compare_paths:
sys.exit(0)
for path in delete_paths:
dst_abs = os.path.join(args.dst_dir, path)
if os.path.exists(dst_abs):
if os.path.isdir(dst_abs):
shutil.rmtree(dst_abs, True)
else:
os.unlink(dst_abs)
for path in copy_paths:
preprocess(path, new_state[path]['preprocessors'])
changes = bool(delete_paths or copy_paths)
for path in compare_paths:
if old_state[path]['type'] != new_state[path]['type'] or \
old_state[path]['preprocessors'] != \
new_state[path]['preprocessors'] or \
old_state[path]['mtime'] < new_state[path]['mtime']:
changes = True
dst_abs = os.path.join(args.dst_dir, path)
if os.path.exists(dst_abs):
if os.path.isdir(dst_abs):
shutil.rmtree(dst_abs, True)
else:
os.unlink(dst_abs)
preprocess(path, new_state[path]['preprocessors'])
if changes:
save_state(args.dst_dir, new_state)
if __name__ == '__main__':
main()

View File

@ -0,0 +1 @@
heatshrink2>=0.12.0

8
reference/zeroify.py Normal file
View File

@ -0,0 +1,8 @@
#!/usr/bin/env python
import sys
if __name__ == '__main__':
data = sys.stdin.buffer.read()
sys.stdout.buffer.write(data + b'\0')