![[previous]](../icons/n_left.png){kind=icon}
![[next]](../icons/n_right.png){kind=icon}
![[first]](../icons/first.png){kind=icon}
![[last]](../icons/last.png){kind=icon}
![[top]](../icons/n_top.png){kind=icon}
![[bottom]](../icons/bottom.png){kind=icon}
![[index]](../icons/index.png){kind=icon}
![[help]](../icons/help.png){kind=icon}
*/
1 /*
2 Virtual File System: GNU Tar file system.
3
4 Copyright (C) 2003-2025
5 Free Software Foundation, Inc.
6
7 Written by:
8 Andrew Borodin <aborodin@vmail.ru>, 2023
9
10 This file is part of the Midnight Commander.
11
12 The Midnight Commander is free software: you can redistribute it
13 and/or modify it under the terms of the GNU General Public License as
14 published by the Free Software Foundation, either version 3 of the License,
15 or (at your option) any later version.
16
17 The Midnight Commander is distributed in the hope that it will be useful,
18 but WITHOUT ANY WARRANTY; without even the implied warranty of
19 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 GNU General Public License for more details.
21
22 You should have received a copy of the GNU General Public License
23 along with this program. If not, see <https://www.gnu.org/licenses/>.
24 */
25
26 /**
27 * \file
28 * \brief Source: Virtual File System: GNU Tar file system
29 */
30
31 /*
32 * Avoid following error:
33 * comparison of unsigned expression < 0 is always false [-Werror=type-limits]
34 *
35 * https://www.boost.org/doc/libs/1_55_0/libs/integer/test/cstdint_test.cpp
36 * We can't suppress this warning on the command line as not all GCC versions support
37 * -Wno-type-limits
38 */
39 #if defined(__GNUC__) && (__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ >= 4))
40 # pragma GCC diagnostic ignored "-Wtype-limits"
41 #endif
42
43 #include <config.h>
44
45 #include <inttypes.h> // uintmax_t
46
47 #include "lib/global.h"
48
49 #include "tar-internal.h"
50
51 /* Old GNU Format.
52 The sparse file information is stored in the oldgnu_header in the following manner:
53
54 The header is marked with type 'S'. Its 'size' field contains the cumulative size
55 of all non-empty blocks of the file. The actual file size is stored in `realsize'
56 member of oldgnu_header.
57
58 The map of the file is stored in a list of 'struct sparse'. Each struct contains
59 offset to the block of data and its size (both as octal numbers). The first file
60 header contains at most 4 such structs (SPARSES_IN_OLDGNU_HEADER). If the map
61 contains more structs, then the field 'isextended' of the main header is set to
62 1 (binary) and the 'struct sparse_header' header follows, containing at most
63 21 following structs (SPARSES_IN_SPARSE_HEADER). If more structs follow, 'isextended'
64 field of the extended header is set and next next extension header follows, etc...
65 */
66
67 /*** global variables ****************************************************************************/
68
69 /*** file scope macro definitions ****************************************************************/
70
71 /* Bound on length of the string representing an unsigned integer
72 value representable in B bits. log10 (2.0) < 146/485. The
73 smallest value of B where this bound is not tight is 2621. */
74 #define INT_BITS_STRLEN_BOUND(b) (((b) * 146 + 484) / 485)
75
76 /* Bound on buffer size needed to represent an integer type or expression T,
77 including the terminating null. T must not be a bit-field expression. */
78 #define INT_BUFSIZE_BOUND(t) (INT_STRLEN_BOUND (t) + 1)
79
80 #define UINTMAX_STRSIZE_BOUND INT_BUFSIZE_BOUND (uintmax_t)
81
82 #define COPY_BUF(arch, b, buf, src) \
83 do \
84 { \
85 char *endp = b->buffer + BLOCKSIZE; \
86 char *dst = buf; \
87 do \
88 { \
89 if (dst == buf + UINTMAX_STRSIZE_BOUND - 1) \
90 /* numeric overflow in sparse archive member */ \
91 return FALSE; \
92 if (src == endp) \
93 { \
94 tar_set_next_block_after (b); \
95 b = tar_find_next_block (arch); \
96 if (b == NULL) \
97 /* unexpected EOF in archive */ \
98 return FALSE; \
99 src = b->buffer; \
100 endp = b->buffer + BLOCKSIZE; \
101 } \
102 *dst = *src++; \
103 } \
104 while (*dst++ != '\n'); \
105 dst[-1] = '\0'; \
106 } \
107 while (FALSE)
108
109 /*** file scope type declarations ****************************************************************/
110
111 struct tar_sparse_file;
112
113 struct tar_sparse_optab
114 {
115 gboolean (*init) (struct tar_sparse_file *file);
116 gboolean (*done) (struct tar_sparse_file *file);
117 gboolean (*sparse_member_p) (struct tar_sparse_file *file);
118 gboolean (*fixup_header) (struct tar_sparse_file *file);
119 gboolean (*decode_header) (tar_super_t *archive, struct tar_sparse_file *file);
120 };
121
122 struct tar_sparse_file
123 {
124 int fd; // File descriptor
125 off_t dumped_size; // Number of bytes actually written to the archive
126 struct tar_stat_info *stat_info; // Information about the file
127 struct tar_sparse_optab const *optab;
128 void *closure; // Any additional data optab calls might reqiure
129 };
130
131 enum oldgnu_add_status
132 {
133 add_ok,
134 add_finish,
135 add_fail
136 };
137
138 /*** forward declarations (file scope functions) *************************************************/
139
140 static gboolean oldgnu_sparse_member_p (struct tar_sparse_file *file);
141 static gboolean oldgnu_fixup_header (struct tar_sparse_file *file);
142 static gboolean oldgnu_get_sparse_info (tar_super_t *archive, struct tar_sparse_file *file);
143
144 static gboolean star_sparse_member_p (struct tar_sparse_file *file);
145 static gboolean star_fixup_header (struct tar_sparse_file *file);
146 static gboolean star_get_sparse_info (tar_super_t *archive, struct tar_sparse_file *file);
147
148 static gboolean pax_sparse_member_p (struct tar_sparse_file *file);
149 static gboolean pax_decode_header (tar_super_t *archive, struct tar_sparse_file *file);
150
151 /*** file scope variables ************************************************************************/
152
153 static struct tar_sparse_optab const oldgnu_optab = {
154 .init = NULL, // No init function
155 .done = NULL, // No done function
156 .sparse_member_p = oldgnu_sparse_member_p,
157 .fixup_header = oldgnu_fixup_header,
158 .decode_header = oldgnu_get_sparse_info,
159 };
160
161 static struct tar_sparse_optab const star_optab = {
162 .init = NULL, // No init function
163 .done = NULL, // No done function
164 .sparse_member_p = star_sparse_member_p,
165 .fixup_header = star_fixup_header,
166 .decode_header = star_get_sparse_info,
167 };
168
169 /* GNU PAX sparse file format. There are several versions:
170 * 0.0
171
172 The initial version of sparse format used by tar 1.14-1.15.1.
173 The sparse file map is stored in x header:
174
175 GNU.sparse.size Real size of the stored file
176 GNU.sparse.numblocks Number of blocks in the sparse map repeat numblocks time
177 GNU.sparse.offset Offset of the next data block
178 GNU.sparse.numbytes Size of the next data block end repeat
179
180 This has been reported as conflicting with the POSIX specs. The reason is
181 that offsets and sizes of non-zero data blocks were stored in multiple instances
182 of GNU.sparse.offset/GNU.sparse.numbytes variables, whereas POSIX requires the
183 latest occurrence of the variable to override all previous occurrences.
184
185 To avoid this incompatibility two following versions were introduced.
186
187 * 0.1
188
189 Used by tar 1.15.2 -- 1.15.91 (alpha releases).
190
191 The sparse file map is stored in x header:
192
193 GNU.sparse.size Real size of the stored file
194 GNU.sparse.numblocks Number of blocks in the sparse map
195 GNU.sparse.map Map of non-null data chunks. A string consisting of comma-separated
196 values "offset,size[,offset,size]..."
197
198 The resulting GNU.sparse.map string can be *very* long. While POSIX does not impose
199 any limit on the length of a x header variable, this can confuse some tars.
200
201 * 1.0
202
203 Starting from this version, the exact sparse format version is specified explicitly
204 in the header using the following variables:
205
206 GNU.sparse.major Major version
207 GNU.sparse.minor Minor version
208
209 X header keeps the following variables:
210
211 GNU.sparse.name Real file name of the sparse file
212 GNU.sparse.realsize Real size of the stored file (corresponds to the old GNU.sparse.size
213 variable)
214
215 The name field of the ustar header is constructed using the pattern "%d/GNUSparseFile.%p/%f".
216
217 The sparse map itself is stored in the file data block, preceding the actual file data.
218 It consists of a series of octal numbers of arbitrary length, delimited by newlines.
219 The map is padded with nulls to the nearest block boundary.
220
221 The first number gives the number of entries in the map. Following are map entries, each one
222 consisting of two numbers giving the offset and size of the data block it describes.
223
224 The format is designed in such a way that non-posix aware tars and tars not supporting
225 GNU.sparse.* keywords will extract each sparse file in its condensed form with the file map
226 attached and will place it into a separate directory. Then, using a simple program it would be
227 possible to expand the file to its original form even without GNU tar.
228
229 Bu default, v.1.0 archives are created. To use other formats, --sparse-version option is provided.
230 Additionally, v.0.0 can be obtained by deleting GNU.sparse.map from 0.1 format:
231 --sparse-version 0.1 --pax-option delete=GNU.sparse.map
232 */
233
234 static struct tar_sparse_optab const pax_optab = {
235 .init = NULL, // No init function
236 .done = NULL, // No done function
237 .sparse_member_p = pax_sparse_member_p,
238 .fixup_header = NULL, // No fixup_header function
239 .decode_header = pax_decode_header,
240 };
241
242 /* --------------------------------------------------------------------------------------------- */
243 /*** file scope functions ************************************************************************/
244 /* --------------------------------------------------------------------------------------------- */
245
246 static gboolean
247 decode_num (uintmax_t *num, const char *arg, uintmax_t maxval)
/* ![[next]](../icons/right.png)
![[first]](../icons/n_first.png)
![[top]](../icons/top.png)
*/
248 {
249 char *arg_lim = NULL;
250 gboolean overflow = FALSE;
251
252 *num = stoint (arg, &arg_lim, &overflow, 0, maxval);
253 return !(arg_lim == arg || *arg_lim != '\0' || overflow);
254 }
255
256 /* --------------------------------------------------------------------------------------------- */
257
258 static gboolean
259 sparse_select_optab (const tar_super_t *archive, struct tar_sparse_file *file)
/* ![[previous]](../icons/left.png)
*/
260 {
261 switch (archive->type)
262 {
263 case TAR_V7:
264 case TAR_USTAR:
265 return FALSE;
266
267 case TAR_OLDGNU:
268 case TAR_GNU: // FIXME: This one should disappear?
269 file->optab = &oldgnu_optab;
270 break;
271
272 case TAR_POSIX:
273 file->optab = &pax_optab;
274 break;
275
276 case TAR_STAR:
277 file->optab = &star_optab;
278 break;
279
280 default:
281 return FALSE;
282 }
283
284 return TRUE;
285 }
286
287 /* --------------------------------------------------------------------------------------------- */
288
289 static gboolean
290 sparse_init (tar_super_t *archive, struct tar_sparse_file *file)
/* */
291 {
292 memset (file, 0, sizeof (*file));
293
294 if (!sparse_select_optab (archive, file))
295 return FALSE;
296
297 if (file->optab->init != NULL)
298 return file->optab->init (file);
299
300 return TRUE;
301 }
302
303 /* --------------------------------------------------------------------------------------------- */
304
305 static gboolean
306 sparse_done (struct tar_sparse_file *file)
/* */
307 {
308 if (file->optab->done != NULL)
309 return file->optab->done (file);
310
311 return TRUE;
312 }
313
314 /* --------------------------------------------------------------------------------------------- */
315
316 static gboolean
317 sparse_member_p (struct tar_sparse_file *file)
/* */
318 {
319 if (file->optab->sparse_member_p != NULL)
320 return file->optab->sparse_member_p (file);
321
322 return FALSE;
323 }
324
325 /* --------------------------------------------------------------------------------------------- */
326
327 static gboolean
328 sparse_fixup_header (struct tar_sparse_file *file)
/* */
329 {
330 if (file->optab->fixup_header != NULL)
331 return file->optab->fixup_header (file);
332
333 return TRUE;
334 }
335
336 /* --------------------------------------------------------------------------------------------- */
337
338 static gboolean
339 sparse_decode_header (tar_super_t *archive, struct tar_sparse_file *file)
/* */
340 {
341 if (file->optab->decode_header != NULL)
342 return file->optab->decode_header (archive, file);
343
344 return TRUE;
345 }
346
347 /* --------------------------------------------------------------------------------------------- */
348
349 static inline void
350 sparse_add_map (struct tar_stat_info *st, struct sp_array *sp)
/* */
351 {
352 if (st->sparse_map == NULL)
353 st->sparse_map = g_array_sized_new (FALSE, FALSE, sizeof (struct sp_array), 1);
354 g_array_append_val (st->sparse_map, *sp);
355 }
356
357 /* --------------------------------------------------------------------------------------------- */
358
359 /**
360 * Add a sparse item to the sparse file
361 */
362 static enum oldgnu_add_status
363 oldgnu_add_sparse (struct tar_sparse_file *file, struct sparse *s)
/* */
364 {
365 struct sp_array sp;
366 off_t size;
367
368 if (s->numbytes[0] == '\0')
369 return add_finish;
370
371 sp.offset = OFF_FROM_HEADER (s->offset);
372 sp.numbytes = OFF_FROM_HEADER (s->numbytes);
373
374 if (sp.offset < 0 || sp.numbytes < 0 || ckd_add (&size, sp.offset, sp.numbytes)
375 || file->stat_info->stat.st_size < size || file->stat_info->archive_file_size < 0)
376 return add_fail;
377
378 sparse_add_map (file->stat_info, &sp);
379
380 return add_ok;
381 }
382
383 /* --------------------------------------------------------------------------------------------- */
384
385 static gboolean
386 oldgnu_sparse_member_p (struct tar_sparse_file *file)
/* */
387 {
388 (void) file;
389
390 return current_header->header.typeflag == GNUTYPE_SPARSE;
391 }
392
393 /* --------------------------------------------------------------------------------------------- */
394
395 static gboolean
396 oldgnu_fixup_header (struct tar_sparse_file *file)
/* */
397 {
398 /* NOTE! st_size was initialized from the header which actually contains archived size.
399 The following fixes it */
400 off_t realsize;
401
402 realsize = OFF_FROM_HEADER (current_header->oldgnu_header.realsize);
403 file->stat_info->archive_file_size = file->stat_info->stat.st_size;
404 file->stat_info->stat.st_size = MAX (0, realsize);
405
406 return (realsize >= 0);
407 }
408
409 /* --------------------------------------------------------------------------------------------- */
410
411 /**
412 * Convert old GNU format sparse data to internal representation.
413 */
414 static gboolean
415 oldgnu_get_sparse_info (tar_super_t *archive, struct tar_sparse_file *file)
/* */
416 {
417 size_t i;
418 union block *h = current_header;
419 gboolean ext_p;
420 enum oldgnu_add_status rc = add_fail;
421
422 if (file->stat_info->sparse_map != NULL)
423 g_array_set_size (file->stat_info->sparse_map, 0);
424
425 for (i = 0; i < SPARSES_IN_OLDGNU_HEADER; i++)
426 {
427 rc = oldgnu_add_sparse (file, &h->oldgnu_header.sp[i]);
428 if (rc != add_ok)
429 break;
430 }
431
432 for (ext_p = h->oldgnu_header.isextended != 0; rc == add_ok && ext_p;
433 ext_p = h->sparse_header.isextended != 0)
434 {
435 h = tar_find_next_block (archive);
436 if (h == NULL)
437 return FALSE;
438
439 tar_set_next_block_after (h);
440
441 for (i = 0; i < SPARSES_IN_SPARSE_HEADER && rc == add_ok; i++)
442 rc = oldgnu_add_sparse (file, &h->sparse_header.sp[i]);
443 }
444
445 return (rc != add_fail);
446 }
447
448 /* --------------------------------------------------------------------------------------------- */
449
450 static gboolean
451 star_sparse_member_p (struct tar_sparse_file *file)
/* */
452 {
453 (void) file;
454
455 return current_header->header.typeflag == GNUTYPE_SPARSE;
456 }
457
458 /* --------------------------------------------------------------------------------------------- */
459
460 static gboolean
461 star_fixup_header (struct tar_sparse_file *file)
/* */
462 {
463 /* NOTE! st_size was initialized from the header which actually contains archived size.
464 The following fixes it */
465 off_t realsize;
466
467 realsize = OFF_FROM_HEADER (current_header->star_in_header.realsize);
468 file->stat_info->archive_file_size = file->stat_info->stat.st_size;
469 file->stat_info->stat.st_size = MAX (0, realsize);
470
471 return (realsize >= 0);
472 }
473
474 /* --------------------------------------------------------------------------------------------- */
475
476 /**
477 * Convert STAR format sparse data to internal representation
478 */
479 static gboolean
480 star_get_sparse_info (tar_super_t *archive, struct tar_sparse_file *file)
/* */
481 {
482 size_t i;
483 union block *h = current_header;
484 gboolean ext_p = TRUE;
485 enum oldgnu_add_status rc = add_ok;
486
487 if (file->stat_info->sparse_map != NULL)
488 g_array_set_size (file->stat_info->sparse_map, 0);
489
490 if (h->star_in_header.prefix[0] == '\0' && h->star_in_header.sp[0].offset[10] != '\0')
491 {
492 // Old star format
493 for (i = 0; i < SPARSES_IN_STAR_HEADER; i++)
494 {
495 rc = oldgnu_add_sparse (file, &h->star_in_header.sp[i]);
496 if (rc != add_ok)
497 break;
498 }
499
500 ext_p = h->star_in_header.isextended != 0;
501 }
502
503 for (; rc == add_ok && ext_p; ext_p = h->star_ext_header.isextended != 0)
504 {
505 h = tar_find_next_block (archive);
506 if (h == NULL)
507 return FALSE;
508
509 tar_set_next_block_after (h);
510
511 for (i = 0; i < SPARSES_IN_STAR_EXT_HEADER && rc == add_ok; i++)
512 rc = oldgnu_add_sparse (file, &h->star_ext_header.sp[i]);
513
514 file->dumped_size += BLOCKSIZE;
515 }
516
517 return (rc != add_fail);
518 }
519
520 /* --------------------------------------------------------------------------------------------- */
521
522 static gboolean
523 pax_sparse_member_p (struct tar_sparse_file *file)
/* */
524 {
525 return file->stat_info->sparse_map != NULL && file->stat_info->sparse_map->len > 0
526 && file->stat_info->sparse_major > 0;
527 }
528
529 /* --------------------------------------------------------------------------------------------- */
530
531 static gboolean
532 pax_decode_header (tar_super_t *archive, struct tar_sparse_file *file)
/* */
533 {
534 if (file->stat_info->sparse_major > 0)
535 {
536 uintmax_t u;
537 char nbuf[UINTMAX_STRSIZE_BOUND];
538 union block *blk;
539 char *p;
540 size_t sparse_map_len;
541 size_t i;
542 off_t start;
543
544 start = tar_current_block_ordinal (archive);
545 tar_set_next_block_after (current_header);
546 blk = tar_find_next_block (archive);
547 if (blk == NULL)
548 // unexpected EOF in archive
549 return FALSE;
550 p = blk->buffer;
551 COPY_BUF (archive, blk, nbuf, p);
552
553 if (!decode_num (&u, nbuf, SIZE_MAX))
554 {
555 // malformed sparse archive member
556 return FALSE;
557 }
558
559 if (file->stat_info->sparse_map == NULL)
560 file->stat_info->sparse_map =
561 g_array_sized_new (FALSE, FALSE, sizeof (struct sp_array), u);
562 else
563 g_array_set_size (file->stat_info->sparse_map, u);
564
565 sparse_map_len = u;
566
567 for (i = 0; i < sparse_map_len; i++)
568 {
569 struct sp_array sp;
570 off_t size;
571
572 COPY_BUF (archive, blk, nbuf, p);
573 if (!decode_num (&u, nbuf, TYPE_MAXIMUM (off_t)))
574 {
575 // malformed sparse archive member
576 return FALSE;
577 }
578 sp.offset = u;
579 COPY_BUF (archive, blk, nbuf, p);
580 if (!decode_num (&u, nbuf, TYPE_MAXIMUM (off_t)) || ckd_add (&size, sp.offset, u)
581 || file->stat_info->stat.st_size < size)
582 {
583 // malformed sparse archive member
584 return FALSE;
585 }
586 sp.numbytes = u;
587 sparse_add_map (file->stat_info, &sp);
588 }
589
590 tar_set_next_block_after (blk);
591
592 file->dumped_size += BLOCKSIZE * (tar_current_block_ordinal (archive) - start);
593 }
594
595 return TRUE;
596 }
597
598 /* --------------------------------------------------------------------------------------------- */
599 /*** public functions ****************************************************************************/
600 /* --------------------------------------------------------------------------------------------- */
601
602 gboolean
603 tar_sparse_member_p (tar_super_t *archive, struct tar_stat_info *st)
/* */
604 {
605 struct tar_sparse_file file;
606
607 if (!sparse_init (archive, &file))
608 return FALSE;
609
610 file.stat_info = st;
611 return sparse_member_p (&file);
612 }
613
614 /* --------------------------------------------------------------------------------------------- */
615
616 gboolean
617 tar_sparse_fixup_header (tar_super_t *archive, struct tar_stat_info *st)
/* */
618 {
619 struct tar_sparse_file file;
620
621 if (!sparse_init (archive, &file))
622 return FALSE;
623
624 file.stat_info = st;
625 return sparse_fixup_header (&file);
626 }
627
628 /* --------------------------------------------------------------------------------------------- */
629
630 enum dump_status
631 tar_sparse_skip_file (tar_super_t *archive, struct tar_stat_info *st)
/* ![[last]](../icons/n_last.png)
*/
632 {
633 gboolean rc = TRUE;
634 struct tar_sparse_file file;
635
636 if (!sparse_init (archive, &file))
637 return dump_status_not_implemented;
638
639 file.stat_info = st;
640 file.fd = -1;
641
642 rc = sparse_decode_header (archive, &file);
643 (void) tar_skip_file (archive, file.stat_info->archive_file_size - file.dumped_size);
644 return (sparse_done (&file) && rc) ? dump_status_ok : dump_status_short;
645 }
646
647 /* --------------------------------------------------------------------------------------------- */
![[bottom]](../icons/n_bottom.png){kind=icon}
*/