rfc1952.txt 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675
  1. Network Working Group P. Deutsch
  2. Request for Comments: 1952 Aladdin Enterprises
  3. Category: Informational May 1996
  4. GZIP file format specification version 4.3
  5. Status of This Memo
  6. This memo provides information for the Internet community. This memo
  7. does not specify an Internet standard of any kind. Distribution of
  8. this memo is unlimited.
  9. IESG Note:
  10. The IESG takes no position on the validity of any Intellectual
  11. Property Rights statements contained in this document.
  12. Notices
  13. Copyright (c) 1996 L. Peter Deutsch
  14. Permission is granted to copy and distribute this document for any
  15. purpose and without charge, including translations into other
  16. languages and incorporation into compilations, provided that the
  17. copyright notice and this notice are preserved, and that any
  18. substantive changes or deletions from the original are clearly
  19. marked.
  20. A pointer to the latest version of this and related documentation in
  21. HTML format can be found at the URL
  22. <ftp://ftp.uu.net/graphics/png/documents/zlib/zdoc-index.html>.
  23. Abstract
  24. This specification defines a lossless compressed data format that is
  25. compatible with the widely used GZIP utility. The format includes a
  26. cyclic redundancy check value for detecting data corruption. The
  27. format presently uses the DEFLATE method of compression but can be
  28. easily extended to use other compression methods. The format can be
  29. implemented readily in a manner not covered by patents.
  30. Deutsch Informational [Page 1]
  31. RFC 1952 GZIP File Format Specification May 1996
  32. Table of Contents
  33. 1. Introduction ................................................... 2
  34. 1.1. Purpose ................................................... 2
  35. 1.2. Intended audience ......................................... 3
  36. 1.3. Scope ..................................................... 3
  37. 1.4. Compliance ................................................ 3
  38. 1.5. Definitions of terms and conventions used ................. 3
  39. 1.6. Changes from previous versions ............................ 3
  40. 2. Detailed specification ......................................... 4
  41. 2.1. Overall conventions ....................................... 4
  42. 2.2. File format ............................................... 5
  43. 2.3. Member format ............................................. 5
  44. 2.3.1. Member header and trailer ........................... 6
  45. 2.3.1.1. Extra field ................................... 8
  46. 2.3.1.2. Compliance .................................... 9
  47. 3. References .................................................. 9
  48. 4. Security Considerations .................................... 10
  49. 5. Acknowledgements ........................................... 10
  50. 6. Author's Address ........................................... 10
  51. 7. Appendix: Jean-Loup Gailly's gzip utility .................. 11
  52. 8. Appendix: Sample CRC Code .................................. 11
  53. 1. Introduction
  54. 1.1. Purpose
  55. The purpose of this specification is to define a lossless
  56. compressed data format that:
  57. * Is independent of CPU type, operating system, file system,
  58. and character set, and hence can be used for interchange;
  59. * Can compress or decompress a data stream (as opposed to a
  60. randomly accessible file) to produce another data stream,
  61. using only an a priori bounded amount of intermediate
  62. storage, and hence can be used in data communications or
  63. similar structures such as Unix filters;
  64. * Compresses data with efficiency comparable to the best
  65. currently available general-purpose compression methods,
  66. and in particular considerably better than the "compress"
  67. program;
  68. * Can be implemented readily in a manner not covered by
  69. patents, and hence can be practiced freely;
  70. * Is compatible with the file format produced by the current
  71. widely used gzip utility, in that conforming decompressors
  72. will be able to read data produced by the existing gzip
  73. compressor.
  74. Deutsch Informational [Page 2]
  75. RFC 1952 GZIP File Format Specification May 1996
  76. The data format defined by this specification does not attempt to:
  77. * Provide random access to compressed data;
  78. * Compress specialized data (e.g., raster graphics) as well as
  79. the best currently available specialized algorithms.
  80. 1.2. Intended audience
  81. This specification is intended for use by implementors of software
  82. to compress data into gzip format and/or decompress data from gzip
  83. format.
  84. The text of the specification assumes a basic background in
  85. programming at the level of bits and other primitive data
  86. representations.
  87. 1.3. Scope
  88. The specification specifies a compression method and a file format
  89. (the latter assuming only that a file can store a sequence of
  90. arbitrary bytes). It does not specify any particular interface to
  91. a file system or anything about character sets or encodings
  92. (except for file names and comments, which are optional).
  93. 1.4. Compliance
  94. Unless otherwise indicated below, a compliant decompressor must be
  95. able to accept and decompress any file that conforms to all the
  96. specifications presented here; a compliant compressor must produce
  97. files that conform to all the specifications presented here. The
  98. material in the appendices is not part of the specification per se
  99. and is not relevant to compliance.
  100. 1.5. Definitions of terms and conventions used
  101. byte: 8 bits stored or transmitted as a unit (same as an octet).
  102. (For this specification, a byte is exactly 8 bits, even on
  103. machines which store a character on a number of bits different
  104. from 8.) See below for the numbering of bits within a byte.
  105. 1.6. Changes from previous versions
  106. There have been no technical changes to the gzip format since
  107. version 4.1 of this specification. In version 4.2, some
  108. terminology was changed, and the sample CRC code was rewritten for
  109. clarity and to eliminate the requirement for the caller to do pre-
  110. and post-conditioning. Version 4.3 is a conversion of the
  111. specification to RFC style.
  112. Deutsch Informational [Page 3]
  113. RFC 1952 GZIP File Format Specification May 1996
  114. 2. Detailed specification
  115. 2.1. Overall conventions
  116. In the diagrams below, a box like this:
  117. +---+
  118. | | <-- the vertical bars might be missing
  119. +---+
  120. represents one byte; a box like this:
  121. +==============+
  122. | |
  123. +==============+
  124. represents a variable number of bytes.
  125. Bytes stored within a computer do not have a "bit order", since
  126. they are always treated as a unit. However, a byte considered as
  127. an integer between 0 and 255 does have a most- and least-
  128. significant bit, and since we write numbers with the most-
  129. significant digit on the left, we also write bytes with the most-
  130. significant bit on the left. In the diagrams below, we number the
  131. bits of a byte so that bit 0 is the least-significant bit, i.e.,
  132. the bits are numbered:
  133. +--------+
  134. |76543210|
  135. +--------+
  136. This document does not address the issue of the order in which
  137. bits of a byte are transmitted on a bit-sequential medium, since
  138. the data format described here is byte- rather than bit-oriented.
  139. Within a computer, a number may occupy multiple bytes. All
  140. multi-byte numbers in the format described here are stored with
  141. the least-significant byte first (at the lower memory address).
  142. For example, the decimal number 520 is stored as:
  143. 0 1
  144. +--------+--------+
  145. |00001000|00000010|
  146. +--------+--------+
  147. ^ ^
  148. | |
  149. | + more significant byte = 2 x 256
  150. + less significant byte = 8
  151. Deutsch Informational [Page 4]
  152. RFC 1952 GZIP File Format Specification May 1996
  153. 2.2. File format
  154. A gzip file consists of a series of "members" (compressed data
  155. sets). The format of each member is specified in the following
  156. section. The members simply appear one after another in the file,
  157. with no additional information before, between, or after them.
  158. 2.3. Member format
  159. Each member has the following structure:
  160. +---+---+---+---+---+---+---+---+---+---+
  161. |ID1|ID2|CM |FLG| MTIME |XFL|OS | (more-->)
  162. +---+---+---+---+---+---+---+---+---+---+
  163. (if FLG.FEXTRA set)
  164. +---+---+=================================+
  165. | XLEN |...XLEN bytes of "extra field"...| (more-->)
  166. +---+---+=================================+
  167. (if FLG.FNAME set)
  168. +=========================================+
  169. |...original file name, zero-terminated...| (more-->)
  170. +=========================================+
  171. (if FLG.FCOMMENT set)
  172. +===================================+
  173. |...file comment, zero-terminated...| (more-->)
  174. +===================================+
  175. (if FLG.FHCRC set)
  176. +---+---+
  177. | CRC16 |
  178. +---+---+
  179. +=======================+
  180. |...compressed blocks...| (more-->)
  181. +=======================+
  182. 0 1 2 3 4 5 6 7
  183. +---+---+---+---+---+---+---+---+
  184. | CRC32 | ISIZE |
  185. +---+---+---+---+---+---+---+---+
  186. Deutsch Informational [Page 5]
  187. RFC 1952 GZIP File Format Specification May 1996
  188. 2.3.1. Member header and trailer
  189. ID1 (IDentification 1)
  190. ID2 (IDentification 2)
  191. These have the fixed values ID1 = 31 (0x1f, \037), ID2 = 139
  192. (0x8b, \213), to identify the file as being in gzip format.
  193. CM (Compression Method)
  194. This identifies the compression method used in the file. CM
  195. = 0-7 are reserved. CM = 8 denotes the "deflate"
  196. compression method, which is the one customarily used by
  197. gzip and which is documented elsewhere.
  198. FLG (FLaGs)
  199. This flag byte is divided into individual bits as follows:
  200. bit 0 FTEXT
  201. bit 1 FHCRC
  202. bit 2 FEXTRA
  203. bit 3 FNAME
  204. bit 4 FCOMMENT
  205. bit 5 reserved
  206. bit 6 reserved
  207. bit 7 reserved
  208. If FTEXT is set, the file is probably ASCII text. This is
  209. an optional indication, which the compressor may set by
  210. checking a small amount of the input data to see whether any
  211. non-ASCII characters are present. In case of doubt, FTEXT
  212. is cleared, indicating binary data. For systems which have
  213. different file formats for ascii text and binary data, the
  214. decompressor can use FTEXT to choose the appropriate format.
  215. We deliberately do not specify the algorithm used to set
  216. this bit, since a compressor always has the option of
  217. leaving it cleared and a decompressor always has the option
  218. of ignoring it and letting some other program handle issues
  219. of data conversion.
  220. If FHCRC is set, a CRC16 for the gzip header is present,
  221. immediately before the compressed data. The CRC16 consists
  222. of the two least significant bytes of the CRC32 for all
  223. bytes of the gzip header up to and not including the CRC16.
  224. [The FHCRC bit was never set by versions of gzip up to
  225. 1.2.4, even though it was documented with a different
  226. meaning in gzip 1.2.4.]
  227. If FEXTRA is set, optional extra fields are present, as
  228. described in a following section.
  229. Deutsch Informational [Page 6]
  230. RFC 1952 GZIP File Format Specification May 1996
  231. If FNAME is set, an original file name is present,
  232. terminated by a zero byte. The name must consist of ISO
  233. 8859-1 (LATIN-1) characters; on operating systems using
  234. EBCDIC or any other character set for file names, the name
  235. must be translated to the ISO LATIN-1 character set. This
  236. is the original name of the file being compressed, with any
  237. directory components removed, and, if the file being
  238. compressed is on a file system with case insensitive names,
  239. forced to lower case. There is no original file name if the
  240. data was compressed from a source other than a named file;
  241. for example, if the source was stdin on a Unix system, there
  242. is no file name.
  243. If FCOMMENT is set, a zero-terminated file comment is
  244. present. This comment is not interpreted; it is only
  245. intended for human consumption. The comment must consist of
  246. ISO 8859-1 (LATIN-1) characters. Line breaks should be
  247. denoted by a single line feed character (10 decimal).
  248. Reserved FLG bits must be zero.
  249. MTIME (Modification TIME)
  250. This gives the most recent modification time of the original
  251. file being compressed. The time is in Unix format, i.e.,
  252. seconds since 00:00:00 GMT, Jan. 1, 1970. (Note that this
  253. may cause problems for MS-DOS and other systems that use
  254. local rather than Universal time.) If the compressed data
  255. did not come from a file, MTIME is set to the time at which
  256. compression started. MTIME = 0 means no time stamp is
  257. available.
  258. XFL (eXtra FLags)
  259. These flags are available for use by specific compression
  260. methods. The "deflate" method (CM = 8) sets these flags as
  261. follows:
  262. XFL = 2 - compressor used maximum compression,
  263. slowest algorithm
  264. XFL = 4 - compressor used fastest algorithm
  265. OS (Operating System)
  266. This identifies the type of file system on which compression
  267. took place. This may be useful in determining end-of-line
  268. convention for text files. The currently defined values are
  269. as follows:
  270. Deutsch Informational [Page 7]
  271. RFC 1952 GZIP File Format Specification May 1996
  272. 0 - FAT filesystem (MS-DOS, OS/2, NT/Win32)
  273. 1 - Amiga
  274. 2 - VMS (or OpenVMS)
  275. 3 - Unix
  276. 4 - VM/CMS
  277. 5 - Atari TOS
  278. 6 - HPFS filesystem (OS/2, NT)
  279. 7 - Macintosh
  280. 8 - Z-System
  281. 9 - CP/M
  282. 10 - TOPS-20
  283. 11 - NTFS filesystem (NT)
  284. 12 - QDOS
  285. 13 - Acorn RISCOS
  286. 255 - unknown
  287. XLEN (eXtra LENgth)
  288. If FLG.FEXTRA is set, this gives the length of the optional
  289. extra field. See below for details.
  290. CRC32 (CRC-32)
  291. This contains a Cyclic Redundancy Check value of the
  292. uncompressed data computed according to CRC-32 algorithm
  293. used in the ISO 3309 standard and in section 8.1.1.6.2 of
  294. ITU-T recommendation V.42. (See http://www.iso.ch for
  295. ordering ISO documents. See gopher://info.itu.ch for an
  296. online version of ITU-T V.42.)
  297. ISIZE (Input SIZE)
  298. This contains the size of the original (uncompressed) input
  299. data modulo 2^32.
  300. 2.3.1.1. Extra field
  301. If the FLG.FEXTRA bit is set, an "extra field" is present in
  302. the header, with total length XLEN bytes. It consists of a
  303. series of subfields, each of the form:
  304. +---+---+---+---+==================================+
  305. |SI1|SI2| LEN |... LEN bytes of subfield data ...|
  306. +---+---+---+---+==================================+
  307. SI1 and SI2 provide a subfield ID, typically two ASCII letters
  308. with some mnemonic value. Jean-Loup Gailly
  309. <gzip@prep.ai.mit.edu> is maintaining a registry of subfield
  310. IDs; please send him any subfield ID you wish to use. Subfield
  311. IDs with SI2 = 0 are reserved for future use. The following
  312. IDs are currently defined:
  313. Deutsch Informational [Page 8]
  314. RFC 1952 GZIP File Format Specification May 1996
  315. SI1 SI2 Data
  316. ---------- ---------- ----
  317. 0x41 ('A') 0x70 ('P') Apollo file type information
  318. LEN gives the length of the subfield data, excluding the 4
  319. initial bytes.
  320. 2.3.1.2. Compliance
  321. A compliant compressor must produce files with correct ID1,
  322. ID2, CM, CRC32, and ISIZE, but may set all the other fields in
  323. the fixed-length part of the header to default values (255 for
  324. OS, 0 for all others). The compressor must set all reserved
  325. bits to zero.
  326. A compliant decompressor must check ID1, ID2, and CM, and
  327. provide an error indication if any of these have incorrect
  328. values. It must examine FEXTRA/XLEN, FNAME, FCOMMENT and FHCRC
  329. at least so it can skip over the optional fields if they are
  330. present. It need not examine any other part of the header or
  331. trailer; in particular, a decompressor may ignore FTEXT and OS
  332. and always produce binary output, and still be compliant. A
  333. compliant decompressor must give an error indication if any
  334. reserved bit is non-zero, since such a bit could indicate the
  335. presence of a new field that would cause subsequent data to be
  336. interpreted incorrectly.
  337. 3. References
  338. [1] "Information Processing - 8-bit single-byte coded graphic
  339. character sets - Part 1: Latin alphabet No.1" (ISO 8859-1:1987).
  340. The ISO 8859-1 (Latin-1) character set is a superset of 7-bit
  341. ASCII. Files defining this character set are available as
  342. iso_8859-1.* in ftp://ftp.uu.net/graphics/png/documents/
  343. [2] ISO 3309
  344. [3] ITU-T recommendation V.42
  345. [4] Deutsch, L.P.,"DEFLATE Compressed Data Format Specification",
  346. available in ftp://ftp.uu.net/pub/archiving/zip/doc/
  347. [5] Gailly, J.-L., GZIP documentation, available as gzip-*.tar in
  348. ftp://prep.ai.mit.edu/pub/gnu/
  349. [6] Sarwate, D.V., "Computation of Cyclic Redundancy Checks via Table
  350. Look-Up", Communications of the ACM, 31(8), pp.1008-1013.
  351. Deutsch Informational [Page 9]
  352. RFC 1952 GZIP File Format Specification May 1996
  353. [7] Schwaderer, W.D., "CRC Calculation", April 85 PC Tech Journal,
  354. pp.118-133.
  355. [8] ftp://ftp.adelaide.edu.au/pub/rocksoft/papers/crc_v3.txt,
  356. describing the CRC concept.
  357. 4. Security Considerations
  358. Any data compression method involves the reduction of redundancy in
  359. the data. Consequently, any corruption of the data is likely to have
  360. severe effects and be difficult to correct. Uncompressed text, on
  361. the other hand, will probably still be readable despite the presence
  362. of some corrupted bytes.
  363. It is recommended that systems using this data format provide some
  364. means of validating the integrity of the compressed data, such as by
  365. setting and checking the CRC-32 check value.
  366. 5. Acknowledgements
  367. Trademarks cited in this document are the property of their
  368. respective owners.
  369. Jean-Loup Gailly designed the gzip format and wrote, with Mark Adler,
  370. the related software described in this specification. Glenn
  371. Randers-Pehrson converted this document to RFC and HTML format.
  372. 6. Author's Address
  373. L. Peter Deutsch
  374. Aladdin Enterprises
  375. 203 Santa Margarita Ave.
  376. Menlo Park, CA 94025
  377. Phone: (415) 322-0103 (AM only)
  378. FAX: (415) 322-1734
  379. EMail: <ghost@aladdin.com>
  380. Questions about the technical content of this specification can be
  381. sent by email to:
  382. Jean-Loup Gailly <gzip@prep.ai.mit.edu> and
  383. Mark Adler <madler@alumni.caltech.edu>
  384. Editorial comments on this specification can be sent by email to:
  385. L. Peter Deutsch <ghost@aladdin.com> and
  386. Glenn Randers-Pehrson <randeg@alumni.rpi.edu>
  387. Deutsch Informational [Page 10]
  388. RFC 1952 GZIP File Format Specification May 1996
  389. 7. Appendix: Jean-Loup Gailly's gzip utility
  390. The most widely used implementation of gzip compression, and the
  391. original documentation on which this specification is based, were
  392. created by Jean-Loup Gailly <gzip@prep.ai.mit.edu>. Since this
  393. implementation is a de facto standard, we mention some more of its
  394. features here. Again, the material in this section is not part of
  395. the specification per se, and implementations need not follow it to
  396. be compliant.
  397. When compressing or decompressing a file, gzip preserves the
  398. protection, ownership, and modification time attributes on the local
  399. file system, since there is no provision for representing protection
  400. attributes in the gzip file format itself. Since the file format
  401. includes a modification time, the gzip decompressor provides a
  402. command line switch that assigns the modification time from the file,
  403. rather than the local modification time of the compressed input, to
  404. the decompressed output.
  405. 8. Appendix: Sample CRC Code
  406. The following sample code represents a practical implementation of
  407. the CRC (Cyclic Redundancy Check). (See also ISO 3309 and ITU-T V.42
  408. for a formal specification.)
  409. The sample code is in the ANSI C programming language. Non C users
  410. may find it easier to read with these hints:
  411. & Bitwise AND operator.
  412. ^ Bitwise exclusive-OR operator.
  413. >> Bitwise right shift operator. When applied to an
  414. unsigned quantity, as here, right shift inserts zero
  415. bit(s) at the left.
  416. ! Logical NOT operator.
  417. ++ "n++" increments the variable n.
  418. 0xNNN 0x introduces a hexadecimal (base 16) constant.
  419. Suffix L indicates a long value (at least 32 bits).
  420. /* Table of CRCs of all 8-bit messages. */
  421. unsigned long crc_table[256];
  422. /* Flag: has the table been computed? Initially false. */
  423. int crc_table_computed = 0;
  424. /* Make the table for a fast CRC. */
  425. void make_crc_table(void)
  426. {
  427. unsigned long c;
  428. Deutsch Informational [Page 11]
  429. RFC 1952 GZIP File Format Specification May 1996
  430. int n, k;
  431. for (n = 0; n < 256; n++) {
  432. c = (unsigned long) n;
  433. for (k = 0; k < 8; k++) {
  434. if (c & 1) {
  435. c = 0xedb88320L ^ (c >> 1);
  436. } else {
  437. c = c >> 1;
  438. }
  439. }
  440. crc_table[n] = c;
  441. }
  442. crc_table_computed = 1;
  443. }
  444. /*
  445. Update a running crc with the bytes buf[0..len-1] and return
  446. the updated crc. The crc should be initialized to zero. Pre- and
  447. post-conditioning (one's complement) is performed within this
  448. function so it shouldn't be done by the caller. Usage example:
  449. unsigned long crc = 0L;
  450. while (read_buffer(buffer, length) != EOF) {
  451. crc = update_crc(crc, buffer, length);
  452. }
  453. if (crc != original_crc) error();
  454. */
  455. unsigned long update_crc(unsigned long crc,
  456. unsigned char *buf, int len)
  457. {
  458. unsigned long c = crc ^ 0xffffffffL;
  459. int n;
  460. if (!crc_table_computed)
  461. make_crc_table();
  462. for (n = 0; n < len; n++) {
  463. c = crc_table[(c ^ buf[n]) & 0xff] ^ (c >> 8);
  464. }
  465. return c ^ 0xffffffffL;
  466. }
  467. /* Return the CRC of the bytes buf[0..len-1]. */
  468. unsigned long crc(unsigned char *buf, int len)
  469. {
  470. return update_crc(0L, buf, len);
  471. }
  472. Deutsch Informational [Page 12]