From 442cf4f7834077c38320fa7c8aee6ff6d576bfde Mon Sep 17 00:00:00 2001 From: Ruud van Asseldonk Date: Sat, 7 Sep 2019 16:12:07 +0200 Subject: squashfsTools: 4.4dev_20180612 -> 4.4 A new release has been made upstream. Reproducibility issues were fixed in that release, so we no longer need those patches. For a full overview of the changes, see the 4.4-specific readme at [1]. The alignment patch no longer applies cleanly; I disabled it for now, and I will try to restore it in a follow-up commit. [1]: https://github.com/plougher/squashfs-tools/blob/52eb4c279cd283ed9802dd1ceb686560b22ffb67/README-4.4 --- ...ATE_EPOCH-is-set-override-timestamps-with.patch | 90 --------- ...ATE_EPOCH-is-set-also-clamp-content-times.patch | 83 -------- .../0003-remove-frag-deflator-thread.patch | 220 --------------------- pkgs/tools/filesystems/squashfs/default.nix | 17 +- 4 files changed, 6 insertions(+), 404 deletions(-) delete mode 100644 pkgs/tools/filesystems/squashfs/0001-If-SOURCE_DATE_EPOCH-is-set-override-timestamps-with.patch delete mode 100644 pkgs/tools/filesystems/squashfs/0002-If-SOURCE_DATE_EPOCH-is-set-also-clamp-content-times.patch delete mode 100644 pkgs/tools/filesystems/squashfs/0003-remove-frag-deflator-thread.patch (limited to 'pkgs') diff --git a/pkgs/tools/filesystems/squashfs/0001-If-SOURCE_DATE_EPOCH-is-set-override-timestamps-with.patch b/pkgs/tools/filesystems/squashfs/0001-If-SOURCE_DATE_EPOCH-is-set-override-timestamps-with.patch deleted file mode 100644 index 5626800e723c..000000000000 --- a/pkgs/tools/filesystems/squashfs/0001-If-SOURCE_DATE_EPOCH-is-set-override-timestamps-with.patch +++ /dev/null @@ -1,90 +0,0 @@ -From 0ab12a8585373be2de5129e14d979c62e7a90d82 Mon Sep 17 00:00:00 2001 -From: Chris Lamb -Date: Mon, 21 Nov 2016 09:33:05 +0100 -Subject: [PATCH] If SOURCE_DATE_EPOCH is set, override timestamps with that - value. - -See https://reproducible-builds.org/specs/source-date-epoch/ for more -information about this environment variable. - -Based on a patch by Alexander Couzens posted on -https://sourceforge.net/p/squashfs/mailman/message/34673610/ - -Signed-off-by: Chris Lamb ---- - squashfs-tools/mksquashfs.c | 38 ++++++++++++++++++++++++++++++++++++- - 1 file changed, 37 insertions(+), 1 deletion(-) - -diff --git a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c -index c2098bd..b49e956 100644 ---- a/squashfs-tools/mksquashfs.c -+++ b/squashfs-tools/mksquashfs.c -@@ -137,6 +137,9 @@ unsigned int cache_bytes = 0, cache_size = 0, inode_count = 0; - /* inode lookup table */ - squashfs_inode *inode_lookup_table = NULL; - -+/* override filesystem creation time */ -+time_t mkfs_fixed_time = -1; -+ - /* in memory directory data */ - #define I_COUNT_SIZE 128 - #define DIR_ENTRIES 32 -@@ -5104,6 +5107,9 @@ int main(int argc, char *argv[]) - int total_mem = get_default_phys_mem(); - int progress = TRUE; - int force_progress = FALSE; -+ char *source_date_epoch, *endptr; -+ unsigned long long epoch; -+ - struct file_buffer **fragment = NULL; - - if(argc > 1 && strcmp(argv[1], "-version") == 0) { -@@ -5641,6 +5647,36 @@ printOptions: - } - } - -+ /* if SOURCE_DATE_EPOCH is set, use that timestamp for the mkfs time */ -+ source_date_epoch = getenv("SOURCE_DATE_EPOCH"); -+ if(source_date_epoch) { -+ errno = 0; -+ epoch = strtoull(source_date_epoch, &endptr, 10); -+ if((errno == ERANGE && (epoch == ULLONG_MAX || epoch == 0)) -+ || (errno != 0 && epoch == 0)) { -+ ERROR("Environment variable $SOURCE_DATE_EPOCH: " -+ "strtoull: %s\n", strerror(errno)); -+ EXIT_MKSQUASHFS(); -+ } -+ if(endptr == source_date_epoch) { -+ ERROR("Environment variable $SOURCE_DATE_EPOCH: " -+ "No digits were found: %s\n", endptr); -+ EXIT_MKSQUASHFS(); -+ } -+ if(*endptr != '\0') { -+ ERROR("Environment variable $SOURCE_DATE_EPOCH: " -+ "Trailing garbage: %s\n", endptr); -+ EXIT_MKSQUASHFS(); -+ } -+ if(epoch > ULONG_MAX) { -+ ERROR("Environment variable $SOURCE_DATE_EPOCH: " -+ "value must be smaller than or equal to " -+ "%lu but was found to be: %llu \n", ULONG_MAX, epoch); -+ EXIT_MKSQUASHFS(); -+ } -+ mkfs_fixed_time = (time_t)epoch; -+ } -+ - /* - * Some compressors may need the options to be checked for validity - * once all the options have been processed -@@ -5993,7 +6029,7 @@ printOptions: - sBlk.flags = SQUASHFS_MKFLAGS(noI, noD, noF, noX, no_fragments, - always_use_fragments, duplicate_checking, exportable, - no_xattrs, comp_opts); -- sBlk.mkfs_time = time(NULL); -+ sBlk.mkfs_time = mkfs_fixed_time != -1 ? mkfs_fixed_time : time(NULL); - - disable_info(); - --- -2.17.0 - diff --git a/pkgs/tools/filesystems/squashfs/0002-If-SOURCE_DATE_EPOCH-is-set-also-clamp-content-times.patch b/pkgs/tools/filesystems/squashfs/0002-If-SOURCE_DATE_EPOCH-is-set-also-clamp-content-times.patch deleted file mode 100644 index 5002375887fb..000000000000 --- a/pkgs/tools/filesystems/squashfs/0002-If-SOURCE_DATE_EPOCH-is-set-also-clamp-content-times.patch +++ /dev/null @@ -1,83 +0,0 @@ -From 32a07d4156a281084c90a4b78affc8b0b32a26fc Mon Sep 17 00:00:00 2001 -From: intrigeri -Date: Mon, 21 Nov 2016 11:41:28 +0000 -Subject: [PATCH] If SOURCE_DATE_EPOCH is set, also clamp content timestamps - with that value. - -Based on a patch by Alexander Couzens posted on -https://sourceforge.net/p/squashfs/mailman/message/34673610/ ---- - squashfs-tools/mksquashfs.c | 15 ++++++++++++--- - 1 file changed, 12 insertions(+), 3 deletions(-) - -diff --git a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c -index b49e956..9f020bf 100644 ---- a/squashfs-tools/mksquashfs.c -+++ b/squashfs-tools/mksquashfs.c -@@ -137,6 +137,9 @@ unsigned int cache_bytes = 0, cache_size = 0, inode_count = 0; - /* inode lookup table */ - squashfs_inode *inode_lookup_table = NULL; - -+/* clamp all timestamps to SOURCE_DATE_EPOCH */ -+time_t content_clamp_time = -1; -+ - /* override filesystem creation time */ - time_t mkfs_fixed_time = -1; - -@@ -2246,6 +2249,8 @@ restat: - pathname_reader(dir_ent), strerror(errno)); - goto read_err; - } -+ if(content_clamp_time != -1 && buf2.st_mtime >= content_clamp_time) -+ buf2.st_mtime = content_clamp_time; - - if(read_size != buf2.st_size) { - close(file); -@@ -3101,7 +3106,7 @@ void dir_scan(squashfs_inode *inode, char *pathname, - buf.st_mode = S_IRWXU | S_IRWXG | S_IRWXO | S_IFDIR; - buf.st_uid = getuid(); - buf.st_gid = getgid(); -- buf.st_mtime = time(NULL); -+ buf.st_mtime = content_clamp_time != -1 ? content_clamp_time : time(NULL); - buf.st_dev = 0; - buf.st_ino = 0; - dir_ent->inode = lookup_inode2(&buf, PSEUDO_FILE_OTHER, 0); -@@ -3127,6 +3115,8 @@ void dir_scan(squashfs_inode *inode, char *pathname, - /* source directory has disappeared? */ - BAD_ERROR("Cannot stat source directory %s because %s\n", - pathname, strerror(errno)); -+ if(content_clamp_time != -1 && buf.st_mtime >= content_clamp_time) -+ buf.st_mtime = content_clamp_time; - dir_ent->inode = lookup_inode(&buf); - } - -@@ -3365,6 +3372,8 @@ struct dir_info *dir_scan1(char *filename, char *subpath, - free_dir_entry(dir_ent); - continue; - } -+ if(content_clamp_time != -1 && buf.st_mtime >= content_clamp_time) -+ buf.st_mtime = content_clamp_time; - - if((buf.st_mode & S_IFMT) != S_IFREG && - (buf.st_mode & S_IFMT) != S_IFDIR && -@@ -3544,7 +3553,7 @@ void dir_scan2(struct dir_info *dir, struct pseudo *pseudo) - buf.st_gid = pseudo_ent->dev->gid; - buf.st_rdev = makedev(pseudo_ent->dev->major, - pseudo_ent->dev->minor); -- buf.st_mtime = time(NULL); -+ buf.st_mtime = content_clamp_time != -1 ? content_clamp_time : time(NULL); - buf.st_ino = pseudo_ino ++; - - if(pseudo_ent->dev->type == 'd') { -@@ -5674,7 +5683,7 @@ printOptions: - "%lu but was found to be: %llu \n", ULONG_MAX, epoch); - EXIT_MKSQUASHFS(); - } -- mkfs_fixed_time = (time_t)epoch; -+ mkfs_fixed_time = content_clamp_time = (time_t)epoch; - } - - /* --- -2.17.0 - diff --git a/pkgs/tools/filesystems/squashfs/0003-remove-frag-deflator-thread.patch b/pkgs/tools/filesystems/squashfs/0003-remove-frag-deflator-thread.patch deleted file mode 100644 index 4be4b96369a8..000000000000 --- a/pkgs/tools/filesystems/squashfs/0003-remove-frag-deflator-thread.patch +++ /dev/null @@ -1,220 +0,0 @@ -From afc0c76a170bd17cbd29bbec6ae6d2227e398570 Mon Sep 17 00:00:00 2001 -From: Alexander Couzens -Date: Fri, 13 Jan 2017 22:00:37 +0100 -Subject: [PATCH] remove frag_deflator_thread - -frag_deflator_thread compress fragments. -Replace the deflator_thread with a function and -use the function instead of the to_frag queue. ---- - squashfs-tools/info.c | 5 --- - squashfs-tools/mksquashfs.c | 76 +++++++++++++------------------------ - squashfs-tools/mksquashfs.h | 2 +- - squashfs-tools/restore.c | 15 +------- - 4 files changed, 30 insertions(+), 68 deletions(-) - -diff --git a/squashfs-tools/info.c b/squashfs-tools/info.c -index 7968c77..028d578 100644 ---- a/squashfs-tools/info.c -+++ b/squashfs-tools/info.c -@@ -96,11 +96,6 @@ void dump_state() - printf("compressed block queue (deflate thread(s) -> main thread)\n"); - dump_seq_queue(to_main, 0); - -- printf("uncompressed packed fragment queue (main thread -> fragment" -- " deflate thread(s))\n"); -- dump_queue(to_frag); -- -- - printf("locked frag queue (compressed frags waiting while multi-block" - " file is written)\n"); - dump_queue(locked_fragment); -diff --git a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c -index cf48e40..cacf14c 100644 ---- a/squashfs-tools/mksquashfs.c -+++ b/squashfs-tools/mksquashfs.c -@@ -270,10 +270,10 @@ unsigned int sid_count = 0, suid_count = 0, sguid_count = 0; - struct cache *reader_buffer, *fragment_buffer, *reserve_cache; - struct cache *bwriter_buffer, *fwriter_buffer; - struct queue *to_reader, *to_deflate, *to_writer, *from_writer, -- *to_frag, *locked_fragment, *to_process_frag; -+ *locked_fragment, *to_process_frag; - struct seq_queue *to_main; - pthread_t reader_thread, writer_thread, main_thread; --pthread_t *deflator_thread, *frag_deflator_thread, *frag_thread; -+pthread_t *deflator_thread, *frag_thread; - pthread_t *restore_thread = NULL; - pthread_mutex_t fragment_mutex = PTHREAD_MUTEX_INITIALIZER; - pthread_mutex_t pos_mutex = PTHREAD_MUTEX_INITIALIZER; -@@ -323,7 +323,7 @@ struct dir_info *scan1_opendir(char *pathname, char *subpath, int depth); - void write_filesystem_tables(struct squashfs_super_block *sBlk, int nopad); - unsigned short get_checksum_mem(char *buff, int bytes); - void check_usable_phys_mem(int total_mem); -- -+void frag_deflator(struct file_buffer *file_buffer); - - void prep_exit() - { -@@ -1540,7 +1540,7 @@ void write_fragment(struct file_buffer *fragment) - pthread_mutex_lock(&fragment_mutex); - fragment_table[fragment->block].unused = 0; - fragments_outstanding ++; -- queue_put(to_frag, fragment); -+ frag_deflator(fragment); - pthread_cleanup_pop(1); - } - -@@ -2412,51 +2412,34 @@ void *deflator(void *arg) - } - - --void *frag_deflator(void *arg) -+void frag_deflator(struct file_buffer *file_buffer) - { -- void *stream = NULL; -- int res; - -- res = compressor_init(comp, &stream, block_size, 1); -- if(res) -- BAD_ERROR("frag_deflator:: compressor_init failed\n"); -- -- pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); -- -- while(1) { -- int c_byte, compressed_size; -- struct file_buffer *file_buffer = queue_get(to_frag); -- struct file_buffer *write_buffer = -+ int c_byte, compressed_size; -+ struct file_buffer *write_buffer = - cache_get(fwriter_buffer, file_buffer->block); - -- c_byte = mangle2(stream, write_buffer->data, file_buffer->data, -- file_buffer->size, block_size, noF, 1); -- compressed_size = SQUASHFS_COMPRESSED_SIZE_BLOCK(c_byte); -- write_buffer->size = compressed_size; -- pthread_mutex_lock(&fragment_mutex); -- if(fragments_locked == FALSE) { -- fragment_table[file_buffer->block].size = c_byte; -- fragment_table[file_buffer->block].start_block = bytes; -- write_buffer->block = bytes; -- bytes += compressed_size; -- fragments_outstanding --; -- queue_put(to_writer, write_buffer); -- pthread_mutex_unlock(&fragment_mutex); -- TRACE("Writing fragment %lld, uncompressed size %d, " -- "compressed size %d\n", file_buffer->block, -- file_buffer->size, compressed_size); -- } else { -- add_pending_fragment(write_buffer, c_byte, -- file_buffer->block); -- pthread_mutex_unlock(&fragment_mutex); -- } -- cache_block_put(file_buffer); -+ c_byte = mangle2(stream, write_buffer->data, file_buffer->data, -+ file_buffer->size, block_size, noF, 1); -+ compressed_size = SQUASHFS_COMPRESSED_SIZE_BLOCK(c_byte); -+ write_buffer->size = compressed_size; -+ if(fragments_locked == FALSE) { -+ fragment_table[file_buffer->block].size = c_byte; -+ fragment_table[file_buffer->block].start_block = bytes; -+ write_buffer->block = bytes; -+ bytes += compressed_size; -+ fragments_outstanding --; -+ queue_put(to_writer, write_buffer); -+ TRACE("Writing fragment %lld, uncompressed size %d, " -+ "compressed size %d\n", file_buffer->block, -+ file_buffer->size, compressed_size); -+ } else { -+ add_pending_fragment(write_buffer, c_byte, -+ file_buffer->block); - } -- -- pthread_cleanup_pop(0); -+ cache_block_put(file_buffer); - } - -- - struct file_buffer *get_file_buffer() - { - struct file_buffer *file_buffer = seq_queue_get(to_main); -@@ -4257,19 +4240,17 @@ void initialise_threads(int readq, int fragq, int bwriteq, int fwriteq, - multiply_overflow(processors * 3, sizeof(pthread_t))) - BAD_ERROR("Processors too large\n"); - -- deflator_thread = malloc(processors * 3 * sizeof(pthread_t)); -+ deflator_thread = malloc(processors * 2 * sizeof(pthread_t)); - if(deflator_thread == NULL) - MEM_ERROR(); - -- frag_deflator_thread = &deflator_thread[processors]; -- frag_thread = &frag_deflator_thread[processors]; -+ frag_thread = &deflator_thread[processors]; - - to_reader = queue_init(1); - to_deflate = queue_init(reader_size); - to_process_frag = queue_init(reader_size); - to_writer = queue_init(bwriter_size + fwriter_size); - from_writer = queue_init(1); -- to_frag = queue_init(fragment_size); - locked_fragment = queue_init(fragment_size); - to_main = seq_queue_init(); - reader_buffer = cache_init(block_size, reader_size, 0, 0); -@@ -4285,9 +4266,6 @@ void initialise_threads(int readq, int fragq, int bwriteq, int fwriteq, - for(i = 0; i < processors; i++) { - if(pthread_create(&deflator_thread[i], NULL, deflator, NULL)) - BAD_ERROR("Failed to create thread\n"); -- if(pthread_create(&frag_deflator_thread[i], NULL, frag_deflator, -- NULL) != 0) -- BAD_ERROR("Failed to create thread\n"); - if(pthread_create(&frag_thread[i], NULL, frag_thrd, - (void *) destination_file) != 0) - BAD_ERROR("Failed to create thread\n"); -diff --git a/squashfs-tools/mksquashfs.h b/squashfs-tools/mksquashfs.h -index 55708a3..dc5bde4 100644 ---- a/squashfs-tools/mksquashfs.h -+++ b/squashfs-tools/mksquashfs.h -@@ -135,7 +135,7 @@ struct append_file { - extern struct cache *reader_buffer, *fragment_buffer, *reserve_cache; - struct cache *bwriter_buffer, *fwriter_buffer; - extern struct queue *to_reader, *to_deflate, *to_writer, *from_writer, -- *to_frag, *locked_fragment, *to_process_frag; -+ *locked_fragment, *to_process_frag; - extern struct append_file **file_mapping; - extern struct seq_queue *to_main; - extern pthread_mutex_t fragment_mutex, dup_mutex; -diff --git a/squashfs-tools/restore.c b/squashfs-tools/restore.c -index 5e336b3..a7aaf2e 100644 ---- a/squashfs-tools/restore.c -+++ b/squashfs-tools/restore.c -@@ -47,8 +47,8 @@ - #define TRUE 1 - - extern pthread_t reader_thread, writer_thread, main_thread; --extern pthread_t *deflator_thread, *frag_deflator_thread, *frag_thread; --extern struct queue *to_deflate, *to_writer, *to_frag, *to_process_frag; -+extern pthread_t *deflator_thread, *frag_thread; -+extern struct queue *to_deflate, *to_writer, *to_process_frag; - extern struct seq_queue *to_main; - extern void restorefs(); - extern int processors; -@@ -120,17 +120,6 @@ void *restore_thrd(void *arg) - pthread_cancel(main_thread); - pthread_join(main_thread, NULL); - -- /* then flush the main thread to fragment deflator thread(s) -- * queue. The fragment deflator thread(s) will idle -- */ -- queue_flush(to_frag); -- -- /* now kill the fragment deflator thread(s) */ -- for(i = 0; i < processors; i++) -- pthread_cancel(frag_deflator_thread[i]); -- for(i = 0; i < processors; i++) -- pthread_join(frag_deflator_thread[i], NULL); -- - /* - * then flush the main thread/fragment deflator thread(s) - * to writer thread queue. The writer thread will idle --- -2.17.0 - diff --git a/pkgs/tools/filesystems/squashfs/default.nix b/pkgs/tools/filesystems/squashfs/default.nix index 5bd3b27eded1..19a56a9466fc 100644 --- a/pkgs/tools/filesystems/squashfs/default.nix +++ b/pkgs/tools/filesystems/squashfs/default.nix @@ -8,26 +8,21 @@ assert lz4Support -> (lz4 != null); stdenv.mkDerivation { pname = "squashfs"; - version = "4.4dev_20180612"; + version = "4.4"; src = fetchFromGitHub { owner = "plougher"; repo = "squashfs-tools"; - sha256 = "1y53z8dkph3khdyhkmkmy0sg9p1n8czv3vj4l324nj8kxyih3l2c"; - rev = "6e242dc95485ada8d1d0b3dd9346c5243d4a517f"; + sha256 = "0697fv8n6739mcyn57jclzwwbbqwpvjdfkv1qh9s56lvyqnplwaw"; + # Tag "4.4" points to this commit. + rev = "52eb4c279cd283ed9802dd1ceb686560b22ffb67"; }; patches = [ - # These patches ensures that mksquashfs output is reproducible. - # See also https://reproducible-builds.org/docs/system-images/ - # and https://github.com/NixOS/nixpkgs/issues/40144. - ./0001-If-SOURCE_DATE_EPOCH-is-set-override-timestamps-with.patch - ./0002-If-SOURCE_DATE_EPOCH-is-set-also-clamp-content-times.patch - ./0003-remove-frag-deflator-thread.patch - # This patch adds an option to pad filesystems (increasing size) in # exchange for better chunking / binary diff calculation. - ./squashfs-tools-4.4-4k-align.patch + # TODO(ruuda): make this patch apply to the proper 4.4 release. + # ./squashfs-tools-4.4-4k-align.patch ] ++ stdenv.lib.optional stdenv.isDarwin ./darwin.patch; buildInputs = [ zlib xz zstd ] -- cgit 1.4.1