squashfsTools: 4.4dev_20180612 -> 4.4

A new release has been made upstream. Reproducibility issues were fixed
in that release, so we no longer need those patches. For a full overview
of the changes, see the 4.4-specific readme at [1].

The alignment patch no longer applies cleanly; I disabled it for now,
and I will try to restore it in a follow-up commit.

[1]: https://github.com/plougher/squashfs-tools/blob/52eb4c279cd283ed9802dd1ceb686560b22ffb67/README-4.4

4 files changed, 6 insertions, 404 deletions

diff --git a/pkgs/tools/filesystems/squashfs/0001-If-SOURCE_DATE_EPOCH-is-set-override-timestamps-with.patch b/pkgs/tools/filesystems/squashfs/0001-If-SOURCE_DATE_EPOCH-is-set-override-timestamps-with.patch
deleted file mode 100644
index 5626800e723c..000000000000
--- a/pkgs/tools/filesystems/squashfs/0001-If-SOURCE_DATE_EPOCH-is-set-override-timestamps-with.patch
+++ /dev/null
@@ -1,90 +0,0 @@
-From 0ab12a8585373be2de5129e14d979c62e7a90d82 Mon Sep 17 00:00:00 2001
-From: Chris Lamb <lamby@debian.org>
-Date: Mon, 21 Nov 2016 09:33:05 +0100
-Subject: [PATCH] If SOURCE_DATE_EPOCH is set, override timestamps with that
- value.
-
-See https://reproducible-builds.org/specs/source-date-epoch/ for more
-information about this environment variable.
-
-Based on a patch by Alexander Couzens <lynxis@fe...> posted on
-https://sourceforge.net/p/squashfs/mailman/message/34673610/
-
-Signed-off-by: Chris Lamb <lamby@debian.org>
----
- squashfs-tools/mksquashfs.c | 38 ++++++++++++++++++++++++++++++++++++-
- 1 file changed, 37 insertions(+), 1 deletion(-)
-
-diff --git a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c
-index c2098bd..b49e956 100644
---- a/squashfs-tools/mksquashfs.c
-+++ b/squashfs-tools/mksquashfs.c
-@@ -137,6 +137,9 @@ unsigned int cache_bytes = 0, cache_size = 0, inode_count = 0;
- /* inode lookup table */
- squashfs_inode *inode_lookup_table = NULL;
- 
-+/* override filesystem creation time */
-+time_t mkfs_fixed_time = -1;
-+
- /* in memory directory data */
- #define I_COUNT_SIZE		128
- #define DIR_ENTRIES		32
-@@ -5104,6 +5107,9 @@ int main(int argc, char *argv[])
- 	int total_mem = get_default_phys_mem();
- 	int progress = TRUE;
- 	int force_progress = FALSE;
-+	char *source_date_epoch, *endptr;
-+	unsigned long long epoch;
-+
- 	struct file_buffer **fragment = NULL;
- 
- 	if(argc > 1 && strcmp(argv[1], "-version") == 0) {
-@@ -5641,6 +5647,36 @@ printOptions:
- 		}
- 	}
- 
-+	/* if SOURCE_DATE_EPOCH is set, use that timestamp for the mkfs time */
-+	source_date_epoch = getenv("SOURCE_DATE_EPOCH");
-+	if(source_date_epoch) {
-+		errno = 0;
-+		epoch = strtoull(source_date_epoch, &endptr, 10);
-+		if((errno == ERANGE && (epoch == ULLONG_MAX || epoch == 0))
-+				|| (errno != 0 && epoch == 0)) {
-+			ERROR("Environment variable $SOURCE_DATE_EPOCH: "
-+				"strtoull: %s\n", strerror(errno));
-+			EXIT_MKSQUASHFS();
-+		}
-+		if(endptr == source_date_epoch) {
-+			ERROR("Environment variable $SOURCE_DATE_EPOCH: "
-+				"No digits were found: %s\n", endptr);
-+			EXIT_MKSQUASHFS();
-+		}
-+		if(*endptr != '\0') {
-+			ERROR("Environment variable $SOURCE_DATE_EPOCH: "
-+				"Trailing garbage: %s\n", endptr);
-+			EXIT_MKSQUASHFS();
-+		}
-+		if(epoch > ULONG_MAX) {
-+			ERROR("Environment variable $SOURCE_DATE_EPOCH: "
-+				"value must be smaller than or equal to "
-+				"%lu but was found to be: %llu \n", ULONG_MAX, epoch);
-+			EXIT_MKSQUASHFS();
-+		}
-+		mkfs_fixed_time = (time_t)epoch;
-+	}
-+
- 	/*
- 	 * Some compressors may need the options to be checked for validity
- 	 * once all the options have been processed
-@@ -5993,7 +6029,7 @@ printOptions:
-	sBlk.flags = SQUASHFS_MKFLAGS(noI, noD, noF, noX, no_fragments,
-		always_use_fragments, duplicate_checking, exportable,
-		no_xattrs, comp_opts);
--	sBlk.mkfs_time = time(NULL);
-+	sBlk.mkfs_time = mkfs_fixed_time != -1 ? mkfs_fixed_time : time(NULL);
- 
- 	disable_info();
- 
--- 
-2.17.0
-
diff --git a/pkgs/tools/filesystems/squashfs/0002-If-SOURCE_DATE_EPOCH-is-set-also-clamp-content-times.patch b/pkgs/tools/filesystems/squashfs/0002-If-SOURCE_DATE_EPOCH-is-set-also-clamp-content-times.patch
deleted file mode 100644
index 5002375887fb..000000000000
--- a/pkgs/tools/filesystems/squashfs/0002-If-SOURCE_DATE_EPOCH-is-set-also-clamp-content-times.patch
+++ /dev/null
@@ -1,83 +0,0 @@
-From 32a07d4156a281084c90a4b78affc8b0b32a26fc Mon Sep 17 00:00:00 2001
-From: intrigeri <intrigeri@boum.org>
-Date: Mon, 21 Nov 2016 11:41:28 +0000
-Subject: [PATCH] If SOURCE_DATE_EPOCH is set, also clamp content timestamps
- with that value.
-
-Based on a patch by Alexander Couzens <lynxis@fe...> posted on
-https://sourceforge.net/p/squashfs/mailman/message/34673610/
----
- squashfs-tools/mksquashfs.c | 15 ++++++++++++---
- 1 file changed, 12 insertions(+), 3 deletions(-)
-
-diff --git a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c
-index b49e956..9f020bf 100644
---- a/squashfs-tools/mksquashfs.c
-+++ b/squashfs-tools/mksquashfs.c
-@@ -137,6 +137,9 @@ unsigned int cache_bytes = 0, cache_size = 0, inode_count = 0;
- /* inode lookup table */
- squashfs_inode *inode_lookup_table = NULL;
- 
-+/* clamp all timestamps to SOURCE_DATE_EPOCH */
-+time_t content_clamp_time = -1;
-+
- /* override filesystem creation time */
- time_t mkfs_fixed_time = -1;
- 
-@@ -2246,6 +2249,8 @@ restat:
- 			  pathname_reader(dir_ent), strerror(errno));
- 		goto read_err;
- 	}
-+	if(content_clamp_time != -1 && buf2.st_mtime >= content_clamp_time)
-+		buf2.st_mtime = content_clamp_time;
- 
- 	if(read_size != buf2.st_size) {
- 		close(file);
-@@ -3101,7 +3106,7 @@ void dir_scan(squashfs_inode *inode, char *pathname,
- 		buf.st_mode = S_IRWXU | S_IRWXG | S_IRWXO | S_IFDIR;
- 		buf.st_uid = getuid();
- 		buf.st_gid = getgid();
--		buf.st_mtime = time(NULL);
-+		buf.st_mtime = content_clamp_time != -1 ? content_clamp_time : time(NULL);
- 		buf.st_dev = 0;
- 		buf.st_ino = 0;
- 		dir_ent->inode = lookup_inode2(&buf, PSEUDO_FILE_OTHER, 0);
-@@ -3127,6 +3115,8 @@ void dir_scan(squashfs_inode *inode, char *pathname,
-			/* source directory has disappeared? */
-			BAD_ERROR("Cannot stat source directory %s because %s\n",
-				pathname, strerror(errno));
-+		if(content_clamp_time != -1 && buf.st_mtime >= content_clamp_time)
-+			buf.st_mtime = content_clamp_time;
- 		dir_ent->inode = lookup_inode(&buf);
- 	}
- 
-@@ -3365,6 +3372,8 @@ struct dir_info *dir_scan1(char *filename, char *subpath,
- 			free_dir_entry(dir_ent);
- 			continue;
- 		}
-+		if(content_clamp_time != -1 && buf.st_mtime >= content_clamp_time)
-+			buf.st_mtime = content_clamp_time;
- 
- 		if((buf.st_mode & S_IFMT) != S_IFREG &&
- 				(buf.st_mode & S_IFMT) != S_IFDIR &&
-@@ -3544,7 +3553,7 @@ void dir_scan2(struct dir_info *dir, struct pseudo *pseudo)
-		buf.st_gid = pseudo_ent->dev->gid;
-		buf.st_rdev = makedev(pseudo_ent->dev->major,
-			pseudo_ent->dev->minor);
--		buf.st_mtime = time(NULL);
-+		buf.st_mtime = content_clamp_time != -1 ? content_clamp_time : time(NULL);
- 		buf.st_ino = pseudo_ino ++;
- 
- 		if(pseudo_ent->dev->type == 'd') {
-@@ -5674,7 +5683,7 @@ printOptions:
- 				"%lu but was found to be: %llu \n", ULONG_MAX, epoch);
- 			EXIT_MKSQUASHFS();
- 		}
--		mkfs_fixed_time = (time_t)epoch;
-+		mkfs_fixed_time = content_clamp_time = (time_t)epoch;
- 	}
- 
- 	/*
--- 
-2.17.0
-
diff --git a/pkgs/tools/filesystems/squashfs/0003-remove-frag-deflator-thread.patch b/pkgs/tools/filesystems/squashfs/0003-remove-frag-deflator-thread.patch
deleted file mode 100644
index 4be4b96369a8..000000000000
--- a/pkgs/tools/filesystems/squashfs/0003-remove-frag-deflator-thread.patch
+++ /dev/null
@@ -1,220 +0,0 @@
-From afc0c76a170bd17cbd29bbec6ae6d2227e398570 Mon Sep 17 00:00:00 2001
-From: Alexander Couzens <lynxis@fe80.eu>
-Date: Fri, 13 Jan 2017 22:00:37 +0100
-Subject: [PATCH] remove frag_deflator_thread
-
-frag_deflator_thread compress fragments.
-Replace the deflator_thread with a function and
-use the function instead of the to_frag queue.
----
- squashfs-tools/info.c       |  5 ---
- squashfs-tools/mksquashfs.c | 76 +++++++++++++------------------------
- squashfs-tools/mksquashfs.h |  2 +-
- squashfs-tools/restore.c    | 15 +-------
- 4 files changed, 30 insertions(+), 68 deletions(-)
-
-diff --git a/squashfs-tools/info.c b/squashfs-tools/info.c
-index 7968c77..028d578 100644
---- a/squashfs-tools/info.c
-+++ b/squashfs-tools/info.c
-@@ -96,11 +96,6 @@ void dump_state()
- 	printf("compressed block queue (deflate thread(s) -> main thread)\n");
- 	dump_seq_queue(to_main, 0);
- 
--	printf("uncompressed packed fragment queue (main thread -> fragment"
--						" deflate thread(s))\n");
--	dump_queue(to_frag);
--
--
- 	printf("locked frag queue (compressed frags waiting while multi-block"
- 						" file is written)\n");
- 	dump_queue(locked_fragment);
-diff --git a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c
-index cf48e40..cacf14c 100644
---- a/squashfs-tools/mksquashfs.c
-+++ b/squashfs-tools/mksquashfs.c
-@@ -270,10 +270,10 @@ unsigned int sid_count = 0, suid_count = 0, sguid_count = 0;
- struct cache *reader_buffer, *fragment_buffer, *reserve_cache;
- struct cache *bwriter_buffer, *fwriter_buffer;
- struct queue *to_reader, *to_deflate, *to_writer, *from_writer,
--	*to_frag, *locked_fragment, *to_process_frag;
-+	*locked_fragment, *to_process_frag;
- struct seq_queue *to_main;
- pthread_t reader_thread, writer_thread, main_thread;
--pthread_t *deflator_thread, *frag_deflator_thread, *frag_thread;
-+pthread_t *deflator_thread, *frag_thread;
- pthread_t *restore_thread = NULL;
- pthread_mutex_t	fragment_mutex = PTHREAD_MUTEX_INITIALIZER;
- pthread_mutex_t	pos_mutex = PTHREAD_MUTEX_INITIALIZER;
-@@ -323,7 +323,7 @@ struct dir_info *scan1_opendir(char *pathname, char *subpath, int depth);
- void write_filesystem_tables(struct squashfs_super_block *sBlk, int nopad);
- unsigned short get_checksum_mem(char *buff, int bytes);
- void check_usable_phys_mem(int total_mem);
--
-+void frag_deflator(struct file_buffer *file_buffer);
- 
- void prep_exit()
- {
-@@ -1540,7 +1540,7 @@ void write_fragment(struct file_buffer *fragment)
- 	pthread_mutex_lock(&fragment_mutex);
- 	fragment_table[fragment->block].unused = 0;
- 	fragments_outstanding ++;
--	queue_put(to_frag, fragment);
-+	frag_deflator(fragment);
- 	pthread_cleanup_pop(1);
- }
- 
-@@ -2412,51 +2412,34 @@ void *deflator(void *arg)
- }
- 
- 
--void *frag_deflator(void *arg)
-+void frag_deflator(struct file_buffer *file_buffer)
- {
--	void *stream = NULL;
--	int res;
- 
--	res = compressor_init(comp, &stream, block_size, 1);
--	if(res)
--		BAD_ERROR("frag_deflator:: compressor_init failed\n");
--
--	pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex);
--
--	while(1) {
--		int c_byte, compressed_size;
--		struct file_buffer *file_buffer = queue_get(to_frag);
--		struct file_buffer *write_buffer =
-+	int c_byte, compressed_size;
-+	struct file_buffer *write_buffer =
- 			cache_get(fwriter_buffer, file_buffer->block);
- 
--		c_byte = mangle2(stream, write_buffer->data, file_buffer->data,
--			file_buffer->size, block_size, noF, 1);
--		compressed_size = SQUASHFS_COMPRESSED_SIZE_BLOCK(c_byte);
--		write_buffer->size = compressed_size;
--		pthread_mutex_lock(&fragment_mutex);
--		if(fragments_locked == FALSE) {
--			fragment_table[file_buffer->block].size = c_byte;
--			fragment_table[file_buffer->block].start_block = bytes;
--			write_buffer->block = bytes;
--			bytes += compressed_size;
--			fragments_outstanding --;
--			queue_put(to_writer, write_buffer);
--			pthread_mutex_unlock(&fragment_mutex);
--			TRACE("Writing fragment %lld, uncompressed size %d, "
--				"compressed size %d\n", file_buffer->block,
--				file_buffer->size, compressed_size);
--		} else {
--				add_pending_fragment(write_buffer, c_byte,
--					file_buffer->block);
--				pthread_mutex_unlock(&fragment_mutex);
--		}
--		cache_block_put(file_buffer);
-+	c_byte = mangle2(stream, write_buffer->data, file_buffer->data,
-+			 file_buffer->size, block_size, noF, 1);
-+	compressed_size = SQUASHFS_COMPRESSED_SIZE_BLOCK(c_byte);
-+	write_buffer->size = compressed_size;
-+	if(fragments_locked == FALSE) {
-+		fragment_table[file_buffer->block].size = c_byte;
-+		fragment_table[file_buffer->block].start_block = bytes;
-+		write_buffer->block = bytes;
-+		bytes += compressed_size;
-+		fragments_outstanding --;
-+		queue_put(to_writer, write_buffer);
-+		TRACE("Writing fragment %lld, uncompressed size %d, "
-+		      "compressed size %d\n", file_buffer->block,
-+		      file_buffer->size, compressed_size);
-+	} else {
-+		add_pending_fragment(write_buffer, c_byte,
-+				     file_buffer->block);
- 	}
--
--	pthread_cleanup_pop(0);
-+	cache_block_put(file_buffer);
- }
- 
--
- struct file_buffer *get_file_buffer()
- {
- 	struct file_buffer *file_buffer = seq_queue_get(to_main);
-@@ -4257,19 +4240,17 @@ void initialise_threads(int readq, int fragq, int bwriteq, int fwriteq,
- 			multiply_overflow(processors * 3, sizeof(pthread_t)))
- 		BAD_ERROR("Processors too large\n");
- 
--	deflator_thread = malloc(processors * 3 * sizeof(pthread_t));
-+	deflator_thread = malloc(processors * 2 * sizeof(pthread_t));
- 	if(deflator_thread == NULL)
- 		MEM_ERROR();
- 
--	frag_deflator_thread = &deflator_thread[processors];
--	frag_thread = &frag_deflator_thread[processors];
-+	frag_thread = &deflator_thread[processors];
- 
- 	to_reader = queue_init(1);
- 	to_deflate = queue_init(reader_size);
- 	to_process_frag = queue_init(reader_size);
- 	to_writer = queue_init(bwriter_size + fwriter_size);
- 	from_writer = queue_init(1);
--	to_frag = queue_init(fragment_size);
- 	locked_fragment = queue_init(fragment_size);
- 	to_main = seq_queue_init();
- 	reader_buffer = cache_init(block_size, reader_size, 0, 0);
-@@ -4285,9 +4266,6 @@ void initialise_threads(int readq, int fragq, int bwriteq, int fwriteq,
- 	for(i = 0; i < processors; i++) {
- 		if(pthread_create(&deflator_thread[i], NULL, deflator, NULL))
- 			BAD_ERROR("Failed to create thread\n");
--		if(pthread_create(&frag_deflator_thread[i], NULL, frag_deflator,
--				NULL) != 0)
--			BAD_ERROR("Failed to create thread\n");
- 		if(pthread_create(&frag_thread[i], NULL, frag_thrd,
- 				(void *) destination_file) != 0)
- 			BAD_ERROR("Failed to create thread\n");
-diff --git a/squashfs-tools/mksquashfs.h b/squashfs-tools/mksquashfs.h
-index 55708a3..dc5bde4 100644
---- a/squashfs-tools/mksquashfs.h
-+++ b/squashfs-tools/mksquashfs.h
-@@ -135,7 +135,7 @@ struct append_file {
- extern struct cache *reader_buffer, *fragment_buffer, *reserve_cache;
- struct cache *bwriter_buffer, *fwriter_buffer;
- extern struct queue *to_reader, *to_deflate, *to_writer, *from_writer,
--	*to_frag, *locked_fragment, *to_process_frag;
-+	*locked_fragment, *to_process_frag;
- extern struct append_file **file_mapping;
- extern struct seq_queue *to_main;
- extern pthread_mutex_t fragment_mutex, dup_mutex;
-diff --git a/squashfs-tools/restore.c b/squashfs-tools/restore.c
-index 5e336b3..a7aaf2e 100644
---- a/squashfs-tools/restore.c
-+++ b/squashfs-tools/restore.c
-@@ -47,8 +47,8 @@
- #define TRUE 1
- 
- extern pthread_t reader_thread, writer_thread, main_thread;
--extern pthread_t *deflator_thread, *frag_deflator_thread, *frag_thread;
--extern struct queue *to_deflate, *to_writer, *to_frag, *to_process_frag;
-+extern pthread_t *deflator_thread, *frag_thread;
-+extern struct queue *to_deflate, *to_writer, *to_process_frag;
- extern struct seq_queue *to_main;
- extern void restorefs();
- extern int processors;
-@@ -120,17 +120,6 @@ void *restore_thrd(void *arg)
- 		pthread_cancel(main_thread);
- 		pthread_join(main_thread, NULL);
- 
--		/* then flush the main thread to fragment deflator thread(s)
--		 * queue.  The fragment deflator thread(s) will idle
--		 */
--		queue_flush(to_frag);
--
--		/* now kill the fragment deflator thread(s) */
--		for(i = 0; i < processors; i++)
--			pthread_cancel(frag_deflator_thread[i]);
--		for(i = 0; i < processors; i++)
--			pthread_join(frag_deflator_thread[i], NULL);
--
- 		/*
- 		 * then flush the main thread/fragment deflator thread(s)
- 		 * to writer thread queue.  The writer thread will idle
--- 
-2.17.0
-
diff --git a/pkgs/tools/filesystems/squashfs/default.nix b/pkgs/tools/filesystems/squashfs/default.nix
index 5bd3b27eded1..19a56a9466fc 100644
--- a/pkgs/tools/filesystems/squashfs/default.nix
+++ b/pkgs/tools/filesystems/squashfs/default.nix
@@ -8,26 +8,21 @@ assert lz4Support -> (lz4 != null);
 
 stdenv.mkDerivation {
   pname = "squashfs";
-  version = "4.4dev_20180612";
+  version = "4.4";
 
   src = fetchFromGitHub {
     owner = "plougher";
     repo = "squashfs-tools";
-    sha256 = "1y53z8dkph3khdyhkmkmy0sg9p1n8czv3vj4l324nj8kxyih3l2c";
-    rev = "6e242dc95485ada8d1d0b3dd9346c5243d4a517f";
+    sha256 = "0697fv8n6739mcyn57jclzwwbbqwpvjdfkv1qh9s56lvyqnplwaw";
+    # Tag "4.4" points to this commit.
+    rev = "52eb4c279cd283ed9802dd1ceb686560b22ffb67";
   };
 
   patches = [
-    # These patches ensures that mksquashfs output is reproducible.
-    # See also https://reproducible-builds.org/docs/system-images/
-    # and https://github.com/NixOS/nixpkgs/issues/40144.
-    ./0001-If-SOURCE_DATE_EPOCH-is-set-override-timestamps-with.patch
-    ./0002-If-SOURCE_DATE_EPOCH-is-set-also-clamp-content-times.patch
-    ./0003-remove-frag-deflator-thread.patch
-
     # This patch adds an option to pad filesystems (increasing size) in
     # exchange for better chunking / binary diff calculation.
-    ./squashfs-tools-4.4-4k-align.patch
+    # TODO(ruuda): make this patch apply to the proper 4.4 release.
+    # ./squashfs-tools-4.4-4k-align.patch
   ] ++ stdenv.lib.optional stdenv.isDarwin ./darwin.patch;
 
   buildInputs = [ zlib xz zstd ]