Update to 0.12

Author: mattfbacon

Cargo.lock

@@ -15,16 +15,16 @@ The bot uses two binaries:
 
 To set up the working environment, create a directory with the following items:
 
-- `fonts`: Copied from the repo. ~~Make sure you have Git LFS set up so the fonts are downloaded properly.~~ The fonts no longer use LFS due to GitHub's new transfer limits.
 - `worker`: The worker binary, copied/hardlinked from the target directory after building.
 - `bot`: The bot binary, copied/hardlinked from the target directory after building. (This doesn't need to be in this directory, but having everything in one place simplifies things.)
 - `db.sqlite`: You can just `touch` this, but the bot needs to be able to write to it.
+(Legacy note: you don't need `fonts` anymore because we use `typst-assets` now.)
 
 To run, CD into this directory, set `DISCORD_TOKEN` to your bot token, set `CACHE_DIRECTORY` and `DB_PATH` to suitable locations, and run the `bot` binary (not the `worker` binary that's also in the directory).
 
 ### Docker
 
-There is a `Dockerfile` and `docker-compose.yml` for running the bot inside a Docker container. 
+There is a `Dockerfile` and `docker-compose.yml` for running the bot inside a Docker container.
 
 To set up the bot with Docker, create a `.env` file like the following:
 

README.md

@@ -1,9 +1,7 @@
 use std::io::Cursor;
 
 use protocol::Rendered;
-use typst::eval::Tracer;
 use typst::layout::{Axis, Size};
-use typst::visualize::Color;
 
 use crate::diagnostic::format_diagnostics;
 use crate::sandbox::Sandbox;
@@ -55,22 +53,21 @@ const BYTES_LIMIT: usize = 25 * 1024 * 1024;
 pub fn render(sandbox: &Sandbox, source: String) -> Result<Rendered, String> {
 	let world = sandbox.with_source(source);
 
-	let mut tracer = Tracer::default();
-	let document =
-		typst::compile(&world, &mut tracer).map_err(|diags| format_diagnostics(&world, &diags))?;
-	let warnings = tracer.warnings();
+	let document = typst::compile(&world);
+	let warnings = document.warnings;
+	let document = document
+		.output
+		.map_err(|diags| format_diagnostics(&world, &diags))?;
 
-	let transparent = Color::from_u8(0, 0, 0, 0);
 	let mut total_attachment_size = 0;
 
 	let images = document
 		.pages
 		.iter()
 		.take(PAGE_LIMIT)
 		.map(|page| {
-			let frame = &page.frame;
-			let pixels_per_point = determine_pixels_per_point(frame.size()).map_err(to_string)?;
-			let pixmap = typst_render::render(frame, pixels_per_point, transparent);
+			let pixels_per_point = determine_pixels_per_point(page.frame.size()).map_err(to_string)?;
+			let pixmap = typst_render::render(page, pixels_per_point);
 
 			let mut writer = Cursor::new(Vec::new());
 

crates/worker/src/render.rs

@@ -1,13 +1,13 @@
-use std::cell::{RefCell, RefMut};
 use std::collections::HashMap;
 use std::path::PathBuf;
+use std::sync::Mutex;
 
-use comemo::Prehashed;
 use typst::diag::{eco_format, FileError, FileResult, PackageError, PackageResult};
 use typst::foundations::{Bytes, Datetime};
 use typst::syntax::package::PackageSpec;
 use typst::syntax::{FileId, Source};
 use typst::text::{Font, FontBook};
+use typst::utils::LazyHash;
 use typst::Library;
 
 struct FileEntry {
@@ -33,27 +33,22 @@ impl FileEntry {
 }
 
 pub struct Sandbox {
-	library: Prehashed<Library>,
-	book: Prehashed<FontBook>,
+	library: LazyHash<Library>,
+	book: LazyHash<FontBook>,
 	fonts: Vec<Font>,
 
 	cache_directory: PathBuf,
 	http: ureq::Agent,
-	files: RefCell<HashMap<FileId, FileEntry>>,
+	files: Mutex<HashMap<FileId, FileEntry>>,
 }
 
 fn fonts() -> Vec<Font> {
-	std::fs::read_dir("fonts")
-		.unwrap()
-		.map(Result::unwrap)
-		.flat_map(|entry| {
-			let path = entry.path();
-			let bytes = std::fs::read(&path).unwrap();
-			let buffer = Bytes::from(bytes);
+	typst_assets::fonts()
+		.flat_map(|bytes| {
+			let buffer = Bytes::from_static(bytes);
 			let face_count = ttf_parser::fonts_in_collection(&buffer).unwrap_or(1);
 			(0..face_count).map(move |face| {
-				Font::new(buffer.clone(), face)
-					.unwrap_or_else(|| panic!("failed to load font from {path:?} (face index {face})"))
+				Font::new(buffer.clone(), face).expect("failed to load font from typst-assets")
 			})
 		})
 		.collect()
@@ -91,15 +86,15 @@ impl Sandbox {
 		let fonts = fonts();
 
 		Self {
-			library: Prehashed::new(Library::default()),
-			book: Prehashed::new(FontBook::from_fonts(&fonts)),
+			library: LazyHash::new(Library::default()),
+			book: LazyHash::new(FontBook::from_fonts(&fonts)),
 			fonts,
 
 			cache_directory: std::env::var_os("CACHE_DIRECTORY")
 				.expect("need the `CACHE_DIRECTORY` env var")
 				.into(),
 			http: ureq::Agent::new(),
-			files: RefCell::new(HashMap::new()),
+			files: Mutex::new(HashMap::new()),
 		}
 	}
 
@@ -163,10 +158,14 @@ impl Sandbox {
 		Ok(path)
 	}
 
-	fn file(&self, id: FileId) -> FileResult<RefMut<'_, FileEntry>> {
-		if let Ok(entry) = RefMut::filter_map(self.files.borrow_mut(), |files| files.get_mut(&id)) {
-			return Ok(entry);
+	// Weird pattern because mapping a MutexGuard is not stable yet.
+	fn file<T>(&self, id: FileId, map: impl FnOnce(&mut FileEntry) -> T) -> FileResult<T> {
+		let mut files = self.files.lock().unwrap();
+		if let Some(entry) = files.get_mut(&id) {
+			return Ok(map(entry));
 		}
+		// `files` must stay locked here so we don't download the same package multiple times.
+		// TODO proper multithreading, maybe with typst-kit.
 
 		'x: {
 			if let Some(package) = id.package() {
@@ -175,12 +174,11 @@ impl Sandbox {
 					break 'x;
 				};
 				let contents = std::fs::read(&path).map_err(|error| FileError::from_io(error, &path))?;
-				return Ok(RefMut::map(self.files.borrow_mut(), |files| {
-					files.entry(id).or_insert(FileEntry {
-						bytes: contents.into(),
-						source: None,
-					})
-				}));
+				let entry = files.entry(id).or_insert(FileEntry {
+					bytes: contents.into(),
+					source: None,
+				});
+				return Ok(map(entry));
 			}
 		}
 
@@ -195,23 +193,23 @@ impl WithSource<'_> {
 }
 
 impl typst::World for WithSource<'_> {
-	fn library(&self) -> &Prehashed<Library> {
+	fn library(&self) -> &LazyHash<Library> {
 		&self.sandbox.library
 	}
 
-	fn main(&self) -> Source {
-		self.source.clone()
+	fn main(&self) -> FileId {
+		self.source.id()
 	}
 
 	fn source(&self, id: FileId) -> FileResult<Source> {
 		if id == self.source.id() {
 			Ok(self.source.clone())
 		} else {
-			self.sandbox.file(id)?.source(id)
+			self.sandbox.file(id, |file| file.source(id))?
 		}
 	}
 
-	fn book(&self) -> &Prehashed<FontBook> {
+	fn book(&self) -> &LazyHash<FontBook> {
 		&self.sandbox.book
 	}
 
@@ -220,7 +218,7 @@ impl typst::World for WithSource<'_> {
 	}
 
 	fn file(&self, id: FileId) -> FileResult<Bytes> {
-		self.sandbox.file(id).map(|file| file.bytes.clone())
+		self.sandbox.file(id, |file| file.bytes.clone())
 	}
 
 	fn today(&self, offset: Option<i64>) -> Option<Datetime> {