370 lines
18 KiB
Python
370 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""Stdlib-only prebuilt artifact inspection, document closure and tar verification.
|
|
|
|
Never builds, downloads, launches an input executable, or overwrites an archive.
|
|
"""
|
|
import argparse
|
|
import gzip
|
|
import hashlib
|
|
import html
|
|
from html.parser import HTMLParser
|
|
import json
|
|
from pathlib import Path, PurePosixPath
|
|
import re
|
|
import stat
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
from urllib.parse import unquote, urlsplit
|
|
|
|
|
|
def digest(path):
|
|
with Path(path).open('rb') as stream:
|
|
return hashlib.file_digest(stream, 'sha256').hexdigest()
|
|
|
|
|
|
def ordinary(path):
|
|
path = Path(path).absolute()
|
|
if not path.is_file():
|
|
raise ValueError(f'Expected ordinary file: {path}')
|
|
reject_reparse(path)
|
|
return path
|
|
|
|
|
|
def reject_reparse(path):
|
|
for candidate in (path, *path.parents):
|
|
info = candidate.lstat()
|
|
if stat.S_ISLNK(info.st_mode) or getattr(info, 'st_file_attributes', 0) & getattr(stat, 'FILE_ATTRIBUTE_REPARSE_POINT', 0):
|
|
raise ValueError(f'Symlink/reparse ancestor not permitted in package input: {candidate}')
|
|
|
|
|
|
def local_links(path):
|
|
text = path.read_text(encoding='utf-8-sig')
|
|
if path.suffix.lower() == '.md':
|
|
# Commands/example syntax are not rendered Markdown links.
|
|
text = re.sub(r'(?ms)^\s*(`{3,}|~{3,}).*?^\s*\1\s*$', '', text)
|
|
text = re.sub(r'`[^`\n]*`', '', text)
|
|
links = re.findall(r'!?\[[^\]\n]*\]\(\s*(<[^>]*>|[^\s)]+)(?:\s+[^)]*)?\)', text)
|
|
links += re.findall(r'(?m)^\s*\[[^]\n]+\]:\s*(<[^>]*>|\S+)', text)
|
|
elif path.suffix.lower() == '.rst':
|
|
links = re.findall(r'`[^`\n]*<([^>]+)>`_', text)
|
|
links += re.findall(r'(?m)^\.\.\s+_[^:\n]+:\s*(\S+)', text)
|
|
elif path.suffix.lower() in ('.html', '.htm'):
|
|
links = []
|
|
else:
|
|
return []
|
|
class Links(HTMLParser):
|
|
def handle_starttag(self, tag, attrs):
|
|
if tag == 'base' and any(name == 'href' for name, _ in attrs):
|
|
raise ValueError(f'HTML base URL requires explicit document-closure support: {path}')
|
|
links.extend(value for name, value in attrs if name in ('href', 'src') and value)
|
|
Links().feed(text)
|
|
return [html.unescape(link.strip('<>')) for link in links]
|
|
|
|
|
|
def linked_path(origin, link):
|
|
parsed = urlsplit(link)
|
|
if parsed.scheme or parsed.netloc or not parsed.path:
|
|
return None
|
|
decoded = unquote(parsed.path)
|
|
if '\\' in decoded or decoded.startswith('/'):
|
|
raise ValueError(f'Nonportable document link: {origin}: {link}')
|
|
parts = []
|
|
for part in (PurePosixPath(origin).parent / decoded).parts:
|
|
if part == '..':
|
|
if not parts:
|
|
raise ValueError(f'Document link escapes package: {origin}: {link}')
|
|
parts.pop()
|
|
elif part not in ('', '.'):
|
|
parts.append(part)
|
|
return '/'.join(parts)
|
|
|
|
|
|
def document_inputs(repository, platform):
|
|
repository = Path(repository).resolve()
|
|
raw = subprocess.check_output(['git', '-C', str(repository), 'ls-files', '-z'])
|
|
tracked = set(raw.decode('utf-8').split('\0')) - {''}
|
|
roots = ['LICENSE', 'docs/server.md', 'docs/api.rst', 'docs/api.html',
|
|
'docs/language.rst', 'docs/language.html', f'docs/{platform}-packaging.md',
|
|
'deployment/Dockerfile', 'deployment/Dockerfile.dockerignore',
|
|
'deployment/arcadia-server.service']
|
|
roots += sorted(name for name in tracked if name.startswith('examples/'))
|
|
# Two explicitly authored inputs can be packaged before their first commit.
|
|
permitted = tracked | set(roots) | {'scripts/package-readme.md', 'docs/linux-packaging.md', 'docs/windows-packaging.md'}
|
|
def repository_file(name):
|
|
candidate = ordinary(repository / name)
|
|
candidate.resolve().relative_to(repository)
|
|
return candidate
|
|
sources = {'README.md': repository_file('scripts/package-readme.md')}
|
|
for name in roots:
|
|
sources[name] = repository_file(name)
|
|
queue = list(sources)
|
|
while queue:
|
|
origin = queue.pop()
|
|
for link in local_links(sources[origin]):
|
|
relative = linked_path(origin, link)
|
|
if relative is None or relative in sources:
|
|
continue
|
|
if relative not in permitted:
|
|
raise ValueError(f'Untracked/missing package document link: {origin}: {link}')
|
|
candidate = repository_file(relative)
|
|
sources[relative] = candidate
|
|
queue.append(relative)
|
|
return {'files': [{'path': name, 'source': str(source)}
|
|
for name, source in sorted(sources.items())]}
|
|
|
|
|
|
def check_documents(stage):
|
|
stage = Path(stage).resolve()
|
|
count = 0
|
|
for source in sorted(stage.rglob('*')):
|
|
if not source.is_file() or source.suffix.lower() not in ('.md', '.rst', '.html', '.htm'):
|
|
continue
|
|
relative = source.relative_to(stage).as_posix()
|
|
# Third-party/Rust source notices can contain their own source-tree links;
|
|
# preserve those upstream texts verbatim, rather than rewriting licenses.
|
|
if relative.startswith('licenses/'):
|
|
continue
|
|
for link in local_links(source):
|
|
target = linked_path(relative, link)
|
|
if target is not None:
|
|
ordinary(stage / target)
|
|
count += 1
|
|
return {'checked_local_document_links': count}
|
|
|
|
|
|
def inspect_elf(path):
|
|
path = ordinary(path)
|
|
with path.open('rb') as stream:
|
|
data = stream.read()
|
|
if len(data) < 64 or data[:6] != b'\x7fELF\x02\x01':
|
|
raise ValueError(f'Expected ELF64 little-endian input: {path}')
|
|
kind, machine = struct.unpack_from('<HH', data, 16)
|
|
if machine != 62 or kind not in (2, 3):
|
|
raise ValueError(f'Expected x86-64 executable/PIE ELF: {path}')
|
|
phoff = struct.unpack_from('<Q', data, 32)[0]
|
|
phsize, phnum = struct.unpack_from('<HH', data, 54)
|
|
if phsize != 56 or not 0 < phnum <= 1024 or phoff + phsize * phnum > len(data):
|
|
raise ValueError('Invalid/bounded ELF program table')
|
|
programs = [struct.unpack_from('<IIQQQQQQ', data, phoff + i * phsize) for i in range(phnum)]
|
|
for _, _, offset, _, _, size, _, _ in programs:
|
|
if offset + size > len(data):
|
|
raise ValueError('ELF segment escapes file')
|
|
|
|
def string(offset, end):
|
|
if not 0 <= offset < end <= len(data):
|
|
raise ValueError('Invalid ELF string range')
|
|
stop = data.find(b'\0', offset, min(end, offset + 4096))
|
|
if stop < 0:
|
|
raise ValueError('Unterminated/bounded ELF string')
|
|
return data[offset:stop].decode('ascii')
|
|
|
|
interpreter = None
|
|
dynamic = []
|
|
for ptype, _, offset, _, _, size, _, _ in programs:
|
|
if ptype == 3:
|
|
interpreter = string(offset, offset + size)
|
|
if ptype == 2:
|
|
if size // 16 > 65536 or size % 16:
|
|
raise ValueError('Invalid/bounded ELF dynamic table')
|
|
for entry in range(offset, offset + size, 16):
|
|
tag, value = struct.unpack_from('<QQ', data, entry)
|
|
if tag == 0:
|
|
break
|
|
dynamic.append((tag, value))
|
|
else:
|
|
raise ValueError('ELF dynamic table lacks terminator')
|
|
tags = dict(dynamic)
|
|
needed, paths = [], []
|
|
if dynamic:
|
|
address, string_size = tags.get(5), tags.get(10)
|
|
if address is None or string_size is None:
|
|
raise ValueError('ELF lacks dynamic string table')
|
|
offsets = [offset + address - vaddr for ptype, _, offset, vaddr, _, size, _, _ in programs
|
|
if ptype == 1 and vaddr <= address and address + string_size <= vaddr + size]
|
|
if len(offsets) != 1:
|
|
raise ValueError('ELF dynamic string table is not uniquely mapped')
|
|
for tag, value in dynamic:
|
|
if tag in (1, 15, 29):
|
|
value = string(offsets[0] + value, offsets[0] + string_size)
|
|
(needed if tag == 1 else paths).append(value)
|
|
if interpreter != '/lib64/ld-linux-x86-64.so.2':
|
|
raise ValueError('Supported Linux package requires the x86-64 glibc dynamic loader')
|
|
if paths:
|
|
raise ValueError(f'ELF RPATH/RUNPATH requires explicit redistribution review: {paths}')
|
|
system = {'libc.so.6', 'libm.so.6', 'libpthread.so.0', 'libdl.so.2', 'librt.so.1',
|
|
'ld-linux-x86-64.so.2', 'libgcc_s.so.1', 'libstdc++.so.6'}
|
|
if set(needed) - system:
|
|
raise ValueError(f'Unreviewed Linux dynamic dependencies: {sorted(set(needed) - system)}')
|
|
versions = sorted(set(match.decode() for match in re.findall(rb'GLIBC_[0-9]+(?:\.[0-9]+)+', data)))
|
|
return {'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data),
|
|
'format': 'ELF64 little-endian x86-64', 'interpreter': interpreter,
|
|
'needed': sorted(set(needed)), 'rpath_runpath': paths,
|
|
'observed_glibc_version_strings': versions,
|
|
'scope': 'Static ELF inspection only. Dynamic loads, display/graphics/portal dependencies and runtime compatibility require native checks.'}
|
|
|
|
|
|
def stage_files(stage):
|
|
stage = Path(stage).absolute()
|
|
reject_reparse(stage)
|
|
stage = stage.resolve()
|
|
files = {}
|
|
for path in sorted(stage.rglob('*')):
|
|
info = path.lstat()
|
|
if stat.S_ISLNK(info.st_mode) or getattr(info, 'st_file_attributes', 0) & getattr(stat, 'FILE_ATTRIBUTE_REPARSE_POINT', 0):
|
|
raise ValueError(f'Symlink not permitted in package: {path}')
|
|
if path.is_file():
|
|
relative = path.relative_to(stage).as_posix()
|
|
if any(c in relative for c in ('\n', '\r', '\\')):
|
|
raise ValueError('Unsafe archive name')
|
|
files[relative] = path
|
|
if not files:
|
|
raise ValueError('Empty package')
|
|
if len({name.casefold() for name in files}) != len(files):
|
|
raise ValueError('Case-colliding package names')
|
|
return files
|
|
|
|
|
|
def make_archive(stage, archive, package_name):
|
|
if not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._-]{0,159}', package_name):
|
|
raise ValueError('Unsafe package name')
|
|
files = stage_files(stage)
|
|
expected = {name: digest(path) for name, path in files.items()}
|
|
with Path(archive).open('xb') as output:
|
|
with gzip.GzipFile(filename='', mode='wb', fileobj=output, mtime=0) as compressed:
|
|
with tarfile.open(fileobj=compressed, mode='w', format=tarfile.PAX_FORMAT) as tar:
|
|
for relative, path in files.items():
|
|
info = tarfile.TarInfo(f'{package_name}/{relative}')
|
|
info.size = path.stat().st_size
|
|
info.mode = 0o755 if relative in ('arcadia-studio', 'arcadia-server') else 0o644
|
|
info.mtime = 0
|
|
with path.open('rb') as source:
|
|
tar.addfile(info, source)
|
|
seen = set()
|
|
with tarfile.open(archive, 'r:gz') as tar:
|
|
for entry in tar:
|
|
prefix = package_name + '/'
|
|
if not entry.isfile() or not entry.name.startswith(prefix):
|
|
raise ValueError('Unexpected archive entry')
|
|
relative = entry.name[len(prefix):]
|
|
if relative in seen or relative not in expected:
|
|
raise ValueError('Duplicate/extra archive entry')
|
|
seen.add(relative)
|
|
stream = tar.extractfile(entry)
|
|
if hashlib.file_digest(stream, 'sha256').hexdigest() != expected[relative]:
|
|
raise ValueError(f'Archive readback mismatch: {relative}')
|
|
mode = 0o755 if relative in ('arcadia-studio', 'arcadia-server') else 0o644
|
|
if entry.mode != mode or entry.uid or entry.gid or entry.mtime:
|
|
raise ValueError('Archive permissions/metadata mismatch')
|
|
if seen != set(expected) or set(stage_files(stage)) != set(expected) or any(digest(path) != expected[name] for name, path in files.items()):
|
|
raise ValueError('Stage changed while archiving or archive incomplete')
|
|
return {'archive': str(Path(archive).resolve()), 'sha256': digest(archive), 'files': len(seen),
|
|
'verification': 'All entries read back, hashes/permissions/metadata checked; stage unchanged.'}
|
|
|
|
|
|
def verify_notices(directory, repository, target):
|
|
directory, repository = Path(directory).resolve(), Path(repository).resolve()
|
|
inventory = json.loads(ordinary(directory / 'dependency-notices.json').read_text(encoding='utf-8-sig'))
|
|
if inventory['target'] != target or inventory['root_packages'] != ['oa-server']:
|
|
raise ValueError('Container requires server-only notices for its exact Linux target')
|
|
if inventory['cargo_lock_sha256'] != digest(repository / 'Cargo.lock'):
|
|
raise ValueError('Container notice Cargo.lock mismatch')
|
|
common = ['Cargo.toml', 'Cargo.lock', 'LICENSE', 'docs/openapi.json']
|
|
common += [path.relative_to(repository).as_posix() for path in sorted((repository / 'build').rglob('*')) if path.is_file()]
|
|
if 'build/windows_manifest.rs' not in common:
|
|
raise ValueError('Container shared build manifest input is missing')
|
|
actual_common = {name: digest(ordinary(repository / name)) for name in common}
|
|
if actual_common != inventory['workspace_build_inputs_sha256']:
|
|
raise ValueError('Container shared build input set/hashes differ from notice inventory')
|
|
current = subprocess.check_output(['rustc', '-Vv'], text=True).splitlines()
|
|
for label in ('release:', 'commit-hash:'):
|
|
if [s for s in current if s.startswith(label)] != [s for s in inventory['rust']['version'] if s.startswith(label)]:
|
|
raise ValueError(f'Container notice compiler mismatch: {label}')
|
|
libdir = Path(subprocess.check_output(['rustc', '--print', 'target-libdir', '--target', target], text=True).strip())
|
|
actual = {path.name: digest(path) for path in sorted(libdir.iterdir()) if path.is_file()}
|
|
if actual != inventory['rust']['target_libraries_sha256']:
|
|
raise ValueError('Container notice target-library hash mismatch')
|
|
checksums = ordinary(directory / 'SHA256SUMS.txt').read_text(encoding='utf-8-sig').splitlines()
|
|
expected = {}
|
|
for line in checksums:
|
|
checksum, name = line.split(' ', 1)
|
|
parts = PurePosixPath(name)
|
|
if parts.is_absolute() or '..' in parts.parts or name in expected or '\\' in name:
|
|
raise ValueError('Unsafe/duplicate notice manifest entry')
|
|
if not re.fullmatch('[0-9a-f]{64}', checksum):
|
|
raise ValueError('Malformed notice checksum')
|
|
expected[name] = checksum
|
|
actual_files = stage_files(directory)
|
|
if set(actual_files) != set(expected) | {'SHA256SUMS.txt'}:
|
|
raise ValueError('Notice bundle inventory differs from checksum manifest')
|
|
for name, checksum in expected.items():
|
|
if digest(ordinary(directory / name)) != checksum:
|
|
raise ValueError(f'Notice bundle file mismatch: {name}')
|
|
def check_notice_references(value):
|
|
if isinstance(value, dict):
|
|
if 'path' in value and 'sha256' in value and value['path'].startswith('licenses/'):
|
|
if expected.get(value['path']) != value['sha256']:
|
|
raise ValueError(f'Inventory notice reference differs from bundle: {value["path"]}')
|
|
for child in value.values():
|
|
check_notice_references(child)
|
|
elif isinstance(value, list):
|
|
for child in value:
|
|
check_notice_references(child)
|
|
check_notice_references(inventory)
|
|
for package in inventory['packages']:
|
|
if not package['source']:
|
|
if package['name'] not in ('oa-core', 'oa-server', 'oa-desktop'):
|
|
raise ValueError('Unexpected workspace package in container notices')
|
|
root = repository / 'crates' / package['name']
|
|
actual_sources = {name: digest(path) for name, path in stage_files(root).items()
|
|
if not name.startswith('target/') and path.name != '.cargo-ok'}
|
|
if actual_sources != package['local_source_sha256']:
|
|
raise ValueError(f'Container workspace differs from notice source set/hashes: {package["name"]}')
|
|
for name, checksum in package['local_source_sha256'].items():
|
|
if PurePosixPath(name).is_absolute() or '..' in PurePosixPath(name).parts or '\\' in name:
|
|
raise ValueError('Unsafe workspace source inventory path')
|
|
if digest(ordinary(root / name)) != checksum:
|
|
raise ValueError(f'Container workspace differs from notice source: {package["name"]}/{name}')
|
|
return {'container_notice_preflight': 'passed', 'target': target}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
commands = parser.add_subparsers(dest='command', required=True)
|
|
docs = commands.add_parser('docs')
|
|
docs.add_argument('--repository', required=True)
|
|
docs.add_argument('--platform', choices=['windows', 'linux'], required=True)
|
|
check = commands.add_parser('check-docs')
|
|
check.add_argument('--stage', required=True)
|
|
elf = commands.add_parser('elf')
|
|
elf.add_argument('path')
|
|
archive = commands.add_parser('archive')
|
|
archive.add_argument('--stage', required=True)
|
|
archive.add_argument('--archive', required=True)
|
|
archive.add_argument('--name', required=True)
|
|
notices = commands.add_parser('verify-notices')
|
|
notices.add_argument('--directory', required=True)
|
|
notices.add_argument('--repository', required=True)
|
|
notices.add_argument('--target', required=True)
|
|
args = parser.parse_args()
|
|
if args.command == 'docs':
|
|
result = document_inputs(args.repository, args.platform)
|
|
elif args.command == 'check-docs':
|
|
result = check_documents(args.stage)
|
|
elif args.command == 'elf':
|
|
result = inspect_elf(args.path)
|
|
elif args.command == 'archive':
|
|
result = make_archive(args.stage, args.archive, args.name)
|
|
else:
|
|
result = verify_notices(args.directory, args.repository, args.target)
|
|
print(json.dumps(result, ensure_ascii=False))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
except (ValueError, OSError, KeyError, subprocess.CalledProcessError, struct.error) as error:
|
|
print(f'Packaging failed: {error}', file=sys.stderr)
|
|
sys.exit(1)
|