Extract zip into folder node via express

I try to find example where I can send a zip (like via postman) and get this zip in my handler and unzip it so specified folder I didn't find much examples for zipping using express I want to unzip it in path web/app

I try something like the following which doesn't works for me , the zip file is not unzipped in the specified folder, any idea what im doing wrong ?

https://nodejs.org/api/zlib.html#zlib_zlib

var zlib = require('zlib'); var fs = require('fs'); const dir = path.join(__dirname, 'web/app/');  if (req.file.mimetype === 'application/zip') {      var unzip = zlib.createUnzip();      var read = fs.createReadStream(req.file);     var write = fs.createWriteStream(dir);     //Transform stream which is unzipping the zipped file     read.pipe(unzip).pipe(write);        console.log("unZipped Successfully");  } 

Any working example will be very helpful, or reference where can I've problem...

while debug I see the that this is when the code failed

var read = fs.createReadStream(req.file);

any idea why?

I've also tried with

var read = fs.createReadStream(req.file.body);

the issue that I dont see the error, reason etc.

when I change it to

var read = fs.createReadStream(req.file.buffer);

the program doesnt exit and i was able to run it until the logger console.log("unZipped Successfully"); but nothing happen...

if there any example with https://www.npmjs.com/package/yauzl yauzl and multer in my context it will be great

update- this is the postman request

4 Answers

Answers 1

Prerequisites:

1. npm i express unzipper multiparty bluebird
2. Create app/web directory in your project root (or you can automate creation if you want).
3. Place all of these files into one directory.
4. Node version that supports async/await (7.6+ as far as I know)

server.js:

const express = require('express'); const Promise = require('bluebird'); const fs = require('fs'); const writeFile = Promise.promisify(fs.writeFile);  const { parseRequest, getFile } = require('./multipart'); const { extractFiles } = require('./zip')  const EXTRACT_DIR = 'web/app';  const app = express();  const uploadFile = async (req, res, next) => {   try {     const body = await parseRequest(req);     const bodyFile = getFile(body, 'file');     if (!/\.zip$/.test(bodyFile.originalFilename)) {       res.status(200).json({ notice: 'not a zip archive, skipping' })       return;     }     const archiveFiles = await extractFiles(bodyFile);      await Promise.each(archiveFiles, async (file) => {       await writeFile(EXTRACT_DIR + '/' + file.path, file.buffer);     })     res.status(200).end();   } catch (e) {     res.status(500).end();   } };  app.post('/files', uploadFile);  app.listen(3000, () => {   console.log('App is listening on port 3000'); }); 

multipart.js

const Promise = require('bluebird'); const { Form } = require('multiparty');  function parseRequest (req, options) {     return new Promise((resolve, reject) => {         const form = new Form(options)         form.parse(req, (err, fields, files) => {             if (err) {                 return reject(err);             }             return resolve({ fields, files });         });     }); }  function getFile (body, field) {     const bodyFile = body.files[field];     const value = bodyFile ? bodyFile[0] : null;     return value || null; }  module.exports = {     parseRequest,     getFile, }; 

zip.js

const unzip = require('unzipper'); const fs = require('fs');  async function extractFiles (file) {     const files = [];     await fs.createReadStream(file.path).pipe(unzip.Parse()).on('entry', async entry => {     // Cleanup system hidden files (or drop this code if not needed)         if (             entry.type !== 'File'             || /^__MACOSX/.test(entry.path)             || /.DS_Store/.test(entry.path)         ) {             entry.autodrain()             return         }         const pathArr = entry.path.split('/');         const path = entry.path;         const buffer = await entry.buffer();         files.push({ buffer, path, originalFilename: pathArr[pathArr.length - 1] });     }).promise();     return files; }  module.exports = {     extractFiles, }; 

Usage:

1. Start a server with node server
2. Send your file in file field in request (key file in postman). Example in curl curl -XPOST -F 'file=@../ttrra-dsp-agency-api/banner.zip' 'localhost:3000/files')

Downsides:

1. Unzipped files are stored in buffer so this method doesn't work great and is not recommended for big archives.

Answers 2

First of all, zlib does not support extracting zip files.

I recommend formidable for handling files because

1. its battle tested
2. most widely used
3. avoids writing boilerplate plate code like reading filestream from request, storing and handling errors
4. easily configurable

Bare minimal solution for your problem with formidable and extract-zip

const express = require('express'); const fs = require('fs'); const extract = require('extract-zip') const formidable = require('formidable'); const path = require('path'); const uploadDir = path.join(__dirname, '/uploads/'); const extractDir = path.join(__dirname, '/app/'); if (!fs.existsSync(uploadDir)) {   fs.mkdirSync(uploadDir); } if (!fs.existsSync(extractDir)) {   fs.mkdirSync(extractDir); }  const server = express();  const uploadMedia = (req, res, next) => {   const form = new formidable.IncomingForm();   // file size limit 100MB. change according to your needs   form.maxFileSize = 100 * 1024 * 1024;   form.keepExtensions = true;   form.multiples = true;   form.uploadDir = uploadDir;    // collect all form files and fileds and pass to its callback   form.parse(req, (err, fields, files) => {     // when form parsing fails throw error     if (err) return res.status(500).json({ error: err });      if (Object.keys(files).length === 0) return res.status(400).json({ message: "no files uploaded" });      // Iterate all uploaded files and get their path, extension, final extraction path     const filesInfo = Object.keys(files).map((key) => {       const file = files[key];       const filePath = file.path;       const fileExt = path.extname(file.name);       const fileName = path.basename(file.name, fileExt);       const destDir = path.join(extractDir, fileName);        return { filePath, fileExt, destDir };     });      // Check whether uploaded files are zip files     const validFiles = filesInfo.every(({ fileExt }) => fileExt === '.zip');      // if uploaded files are not zip files, return error     if (!validFiles) return res.status(400).json({ message: "unsupported file type" });      res.status(200).json({ uploaded: true });      // iterate through each file path and extract them     filesInfo.forEach(({filePath, destDir}) => {       // create directory with timestamp to prevent overwrite same directory names       extract(filePath, { dir: `${destDir}_${new Date().getTime()}` }, (err) => {         if (err) console.error('extraction failed.');       });     });   });    // runs when new file detected in upload stream   form.on('fileBegin', function (name, file) {     // get the file base name `index.css.zip` => `index.html`     const fileName = path.basename(file.name, path.extname(file.name));     const fileExt = path.extname(file.name);     // create files with timestamp to prevent overwrite same file names     file.path = path.join(uploadDir, `${fileName}_${new Date().getTime()}${fileExt}`);   }); }  server.post('/upload', uploadMedia);  server.listen(3000, (err) => {   if (err) throw err; }); 

This solution works for single/multiple file uploads. The one problem with this solution is, wrong file types will get uploaded to uploaded directory though server throw error.

To test with postman:

Answers 3

Without a full example it's tough to say what the real problem is. But according to Express docs it says:

In Express 4, req.files is no longer available on the req object by default. To access uploaded files on the req.files object, use multipart-handling middleware like busboy, multer, formidable, multiparty, connect-multiparty, or pez.

So if you are not using a middleware library to handle uploading files, it's tough to tell what the value of req.file is.

I am also a bit worried that you are trying to use zlib to decompress a zip file, since the library only supports gzip.

The zlib module provides compression functionality implemented using Gzip and Deflate/Inflate

You would want to check for req.file.mimetype === 'application/gzip'

Here are some posts related to unzipping zip files:

• How to download and unzip a zip file in memory in NodeJs?
• Simplest way to download and unzip files in Node.js cross-platform?

Answers 4

This is my code for uploading a file to express server.

//require express library var express = require('express'); //require the express router var router = express.Router(); //require multer for the file uploads var multer = require('multer');  //File Upload  var storage = multer.diskStorage({   // destino del fichero   destination: function (req, file, cb) {     cb(null, './uploads/logo')   },   // renombrar fichero   filename: function (req, file, cb) {     cb(null, file.originalname);   } });  var upload = multer({ storage: storage });  router.post("/", upload.array("uploads[]", 1), function (req, res) {   res.json('Uploaded logo successfully'); });   module.exports = router;  

Read More

Python zip multiple directories into one zip file

I have a top directory ds237 which has multiple sub-directories under it as below:

ds237/ ├── dataset_description.json ├── derivatives ├── sub-01 ├── sub-02 ├── sub-03 ├── sub-04 ├── sub-05 ├── sub-06 ├── sub-07 ├── sub-08 ├── sub-09 ├── sub-10 ├── sub-11 ├── sub-12 ├── sub-13 ├── sub-21 ├── sub-22 ├── sub-23 ├── sub-24 ├── sub-25 ├── sub-26 ├── sub-27 ├── sub-28 ├── sub-29

I am trying to create multiple zip files(with proper zip names) from ds237 as per size of the zip files. sub01-01.zip: contain sub-01 to sub-07 sub08-13.zip : it contains sub08 to sub-13

I have written a logic which creates a list of sub-directories [sub-01,sub-02, sub-03, sub-04, sub-05]. I have created the list so that the total size of the all sub directories in the list should not be > 5gb.

My question: is how can i write a function to zip these sub-dirs (which are in a list) into a destination zip file with proper name. Basically i want to write a function as follows:

def zipit([list of subdirs], 'path/to/zipfile/sub*-*.zip'):

I linux i generally achieve this by: 'zip -r compress/sub01-08.zip ds237/sub-0[1-8]'

3 Answers

Answers 1

Looking at https://stackoverflow.com/a/1855118/375530, you can re-use that answer's function to add a directory to a ZipFile.

import os import zipfile   def zipdir(path, ziph):     # ziph is zipfile handle     for root, dirs, files in os.walk(path):         for file in files:             ziph.write(os.path.join(root, file),                        os.path.relpath(os.path.join(root, file),                                        os.path.join(path, '..')))   def zipit(dir_list, zip_name):     zipf = zipfile.ZipFile(zip_name, 'w', zipfile.ZIP_DEFLATED)     for dir in dir_list:         zipdir(dir, zipf)     zipf.close() 

The zipit function should be called with your pre-chunked list and a given name. You can use string formatting if you want to use a programmatic name (e.g. "path/to/zipfile/sub{}-{}.zip".format(start, end)).

Answers 2

You can use subprocess calling 'zip' and passing the paths as arguments

Answers 3

The following will give you zip file with a first folder ds100:

import os import zipfile      def zipit(folders, zip_filename):     zip_file = zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED)      for folder in folders:         for dirpath, dirnames, filenames in os.walk(folder):             for filename in filenames:                 zip_file.write(                     os.path.join(dirpath, filename),                     os.path.relpath(os.path.join(dirpath, filename), os.path.join(folders[0], '../..')))      zip_file.close()   folders = [     "/Users/aba/ds100/sub-01",     "/Users/aba/ds100/sub-02",     "/Users/aba/ds100/sub-03",     "/Users/aba/ds100/sub-04",     "/Users/aba/ds100/sub-05"]  zipit(folders, "/Users/aba/ds100/sub01-05.zip") 

For example sub01-05.zip would have a structure similar to:

ds100 ├── sub-01 |   ├── 1 |       ├── 2 |   ├── 1 |   ├── 2 ├── sub-02     ├── 1         ├── 2     ├── 1     ├── 2 

Read More

Can we unzip file in FTP server using C#

Can I extract the ZIP file in FTP and place this extracted file on the same location using C#?

2 Answers

Answers 1

It's not possible.

There's no API in the FTP protocol to un-ZIP a file on a server.

---

Though, it's not uncommon that one, in addition to an FTP access, have also an SSH access. If that's the case, you can connect with the SSH and execute the unzip shell command (or similar) on the server to decompress the files.
See C# send a simple SSH command.

If you need, you can then download the extracted files using the FTP protocol (Though if you have the SSH access, you will also have an SFTP access. Then, use the SFTP instead of the FTP.).

---

Some (very few) FTP servers offer an API to execute an arbitrary shell (or other) commands using the SITE EXEC command (or similar). But that's really very rare. You can use this API the same way as the SSH above.

Answers 2

Download via FTP to MemoryStream, then you can unzip, example shows how to get stream, just change to MemoryStream and unzip. Example doesn't use MemoryStream but if you are familiar with streams it should be trivial to modify these two examples to work for you.

example from: https://docs.microsoft.com/en-us/dotnet/framework/network-programming/how-to-download-files-with-ftp

using System;   using System.IO;   using System.Net;   using System.Text;    namespace Examples.System.Net   {       public class WebRequestGetExample       {           public static void Main ()           {               // Get the object used to communicate with the server.               FtpWebRequest request = (FtpWebRequest)WebRequest.Create("ftp://www.contoso.com/test.htm");               request.Method = WebRequestMethods.Ftp.DownloadFile;                // This example assumes the FTP site uses anonymous logon.               request.Credentials = new NetworkCredential ("anonymous","janeDoe@contoso.com");                FtpWebResponse response = (FtpWebResponse)request.GetResponse();                Stream responseStream = response.GetResponseStream();               StreamReader reader = new StreamReader(responseStream);               Console.WriteLine(reader.ReadToEnd());                Console.WriteLine("Download Complete, status {0}", response.StatusDescription);                reader.Close();               response.Close();             }       }   } 

decompress stream, example from: https://docs.microsoft.com/en-us/dotnet/standard/io/how-to-compress-and-extract-files

using System; using System.IO; using System.IO.Compression;  namespace ConsoleApplication {     class Program     {         static void Main(string[] args)         {             using (FileStream zipToOpen = new FileStream(@"c:\users\exampleuser\release.zip", FileMode.Open))             {                 using (ZipArchive archive = new ZipArchive(zipToOpen, ZipArchiveMode.Update))                 {                     ZipArchiveEntry readmeEntry = archive.CreateEntry("Readme.txt");                     using (StreamWriter writer = new StreamWriter(readmeEntry.Open()))                     {                             writer.WriteLine("Information about this package.");                             writer.WriteLine("========================");                     }                 }             }         }     } } 

Read More

node.js compressing ZIP to memory

I want to zip some data into a writableStream.

the purpose is to do all in memory and not to create an actual zip file on disk.

For testing only, i'm creating a ZIP file on disk. But when I try to open output.zip i get the following error: "the archive is either in unknown format or damaged". (WinZip at Windows 7 and also a similar error on MAC)

What am I doing wrong?

const   fs = require('fs'),     archiver = require('archiver'),     streamBuffers = require('stream-buffers');  let outputStreamBuffer = new streamBuffers.WritableStreamBuffer({     initialSize: (1000 * 1024),   // start at 1000 kilobytes.     incrementAmount: (1000 * 1024) // grow by 1000 kilobytes each time buffer overflows. });  let archive = archiver('zip', {     zlib: { level: 9 } // Sets the compression level. }); archive.pipe(outputStreamBuffer);  archive.append("this is a test", { name: "test.txt"}); archive.finalize();  outputStreamBuffer.end();  fs.writeFile('output.zip', outputStreamBuffer.getContents(), function() { console.log('done!'); }); 

1 Answers

Answers 1

You are not waiting for the content to be written to the output stream.

Comment out outputStreamBuffer.end(); from your code and change it to the following...

outputStreamBuffer.on('finish', function () {     fs.writeFile('output.zip', outputStreamBuffer.getContents(), function() {          console.log('done!');     }); }); 

Read More

How to create a very specific zip file structure using Java

I am developing software to integrate into a legacy system, where I send zip files to an FTP server and the legacy system scans periodically dumping the files into a folder, either "Completed" or "Maybe_corrupted".

My files are systematically dumped into the "maybe_corrupted" folder. After investigation it appears that this is due the structure of the zip files.

If I download the file from the FTP onto a Windows computer, I unzip all of the zip files (there are zip files inside zip files) and rezip the files into the exact same configuration the legacy program accepts the files.

Specifically, my question is - how can I parameterize java.util.zip, and what parameters should I modify to bring it closer to the default windows zip utility ?

File comparison

As suggested by Tobias Otto, I have used a file comparator (UltraCompare) to do a comparison between 2 binary files :

• The file originally downloaded from the FTP and rejected by the legacy system (left)
• The decompressed-recompressed file which was accepted (right)

Simply comparing the bytes, it is apparent that the files are not the same. Note that bytes in red are differences, grey are similarities :

Zip details

As suggested by David Duponchel, I have used zipdetails to extract the structure of each file. Obviously this is a very powerful tool but I'm not sure how to interpret the ouput, as stated in the usage guide, the output makes reference to this document.

Original file :

00000 LOCAL HEADER #1       04034B50 00004 Extract Zip Spec      14 '2.0' 00005 Extract OS            00 'MS-DOS' 00006 General Purpose Flag  0808       [Bits 1-2]            0 'Normal Compression'       [Bit  3]              1 'Streamed'       [Bit 11]              1 'Language Encoding' 00008 Compression Method    0008 'Deflated' 0000A Last Mod Time         491E43F7 'Tue Aug 30 08:31:46 2016' 0000E CRC                   00000000 00012 Compressed Length     00000000 00016 Uncompressed Length   00000000 0001A Filename Length       0018 0001C Extra Length          0000 0001E Filename              'Mean.20160830_073000.zip' 00036 PAYLOAD  05002 STREAMING DATA HEADER 08074B50 05006 CRC                   A21A5BC4 0500A Compressed Length     00004FCC 0500E Uncompressed Length   000054D8  05012 LOCAL HEADER #2       04034B50 05016 Extract Zip Spec      14 '2.0' 05017 Extract OS            00 'MS-DOS' 05018 General Purpose Flag  0808       [Bits 1-2]            0 'Normal Compression'       [Bit  3]              1 'Streamed'       [Bit 11]              1 'Language Encoding' 0501A Compression Method    0008 'Deflated' 0501C Last Mod Time         491E43F7 'Tue Aug 30 08:31:46 2016' 05020 CRC                   00000000 05024 Compressed Length     00000000 05028 Uncompressed Length   00000000 0502C Filename Length       0018 0502E Extra Length          0000 05030 Filename              'Mean.20160830_081500.zip' 05048 PAYLOAD  08FFF STREAMING DATA HEADER 08074B50 09003 CRC                   BAE824D6 09007 Compressed Length     00003FB7 0900B Uncompressed Length   000043A3  0900F LOCAL HEADER #3       04034B50 09013 Extract Zip Spec      14 '2.0' 09014 Extract OS            00 'MS-DOS' 09015 General Purpose Flag  0808       [Bits 1-2]            0 'Normal Compression'       [Bit  3]              1 'Streamed'       [Bit 11]              1 'Language Encoding' 09017 Compression Method    0008 'Deflated' 09019 Last Mod Time         491E43F7 'Tue Aug 30 08:31:46 2016' 0901D CRC                   00000000 09021 Compressed Length     00000000 09025 Uncompressed Length   00000000 09029 Filename Length       0018 0902B Extra Length          0000 0902D Filename              'Mean.20160830_071500.zip' 09045 PAYLOAD  0E05E STREAMING DATA HEADER 08074B50 0E062 CRC                   EDC8AE4E 0E066 Compressed Length     00005019 0E06A Uncompressed Length   000054EA  0E06E LOCAL HEADER #4       04034B50 0E072 Extract Zip Spec      14 '2.0' 0E073 Extract OS            00 'MS-DOS' 0E074 General Purpose Flag  0808       [Bits 1-2]            0 'Normal Compression'       [Bit  3]              1 'Streamed'       [Bit 11]              1 'Language Encoding' 0E076 Compression Method    0008 'Deflated' 0E078 Last Mod Time         491E43F7 'Tue Aug 30 08:31:46 2016' 0E07C CRC                   00000000 0E080 Compressed Length     00000000 0E084 Uncompressed Length   00000000 0E088 Filename Length       0018 0E08A Extra Length          0000 0E08C Filename              'Mean.20160830_080000.zip' 0E0A4 PAYLOAD  15444 STREAMING DATA HEADER 08074B50 15448 CRC                   C37437FB 1544C Compressed Length     000073A0 15450 Uncompressed Length   00008054  15454 LOCAL HEADER #5       04034B50 15458 Extract Zip Spec      14 '2.0' 15459 Extract OS            00 'MS-DOS' 1545A General Purpose Flag  0808       [Bits 1-2]            0 'Normal Compression'       [Bit  3]              1 'Streamed'       [Bit 11]              1 'Language Encoding' 1545C Compression Method    0008 'Deflated' 1545E Last Mod Time         491E43F7 'Tue Aug 30 08:31:46 2016' 15462 CRC                   00000000 15466 Compressed Length     00000000 1546A Uncompressed Length   00000000 1546E Filename Length       0018 15470 Extra Length          0000 15472 Filename              'Mean.20160830_070000.zip' 1548A PAYLOAD  19E5D STREAMING DATA HEADER 08074B50 19E61 CRC                   40E52180 19E65 Compressed Length     000049D3 19E69 Uncompressed Length   00005110  19E6D CENTRAL HEADER #1     02014B50 19E71 Created Zip Spec      14 '2.0' 19E72 Created OS            00 'MS-DOS' 19E73 Extract Zip Spec      14 '2.0' 19E74 Extract OS            00 'MS-DOS' 19E75 General Purpose Flag  0808       [Bits 1-2]            0 'Normal Compression'       [Bit  3]              1 'Streamed'       [Bit 11]              1 'Language Encoding' 19E77 Compression Method    0008 'Deflated' 19E79 Last Mod Time         491E43F7 'Tue Aug 30 08:31:46 2016' 19E7D CRC                   A21A5BC4 19E81 Compressed Length     00004FCC 19E85 Uncompressed Length   000054D8 19E89 Filename Length       0018 19E8B Extra Length          0000 19E8D Comment Length        0000 19E8F Disk Start            0000 19E91 Int File Attributes   0000       [Bit 0]               0 'Binary Data' 19E93 Ext File Attributes   00000000 19E97 Local Header Offset   00000000 19E9B Filename              'Mean.20160830_073000.zip'  19EB3 CENTRAL HEADER #2     02014B50 19EB7 Created Zip Spec      14 '2.0' 19EB8 Created OS            00 'MS-DOS' 19EB9 Extract Zip Spec      14 '2.0' 19EBA Extract OS            00 'MS-DOS' 19EBB General Purpose Flag  0808       [Bits 1-2]            0 'Normal Compression'       [Bit  3]              1 'Streamed'       [Bit 11]              1 'Language Encoding' 19EBD Compression Method    0008 'Deflated' 19EBF Last Mod Time         491E43F7 'Tue Aug 30 08:31:46 2016' 19EC3 CRC                   BAE824D6 19EC7 Compressed Length     00003FB7 19ECB Uncompressed Length   000043A3 19ECF Filename Length       0018 19ED1 Extra Length          0000 19ED3 Comment Length        0000 19ED5 Disk Start            0000 19ED7 Int File Attributes   0000       [Bit 0]               0 'Binary Data' 19ED9 Ext File Attributes   00000000 19EDD Local Header Offset   00005012 19EE1 Filename              'Mean.20160830_081500.zip'  19EF9 CENTRAL HEADER #3     02014B50 19EFD Created Zip Spec      14 '2.0' 19EFE Created OS            00 'MS-DOS' 19EFF Extract Zip Spec      14 '2.0' 19F00 Extract OS            00 'MS-DOS' 19F01 General Purpose Flag  0808       [Bits 1-2]            0 'Normal Compression'       [Bit  3]              1 'Streamed'       [Bit 11]              1 'Language Encoding' 19F03 Compression Method    0008 'Deflated' 19F05 Last Mod Time         491E43F7 'Tue Aug 30 08:31:46 2016' 19F09 CRC                   EDC8AE4E 19F0D Compressed Length     00005019 19F11 Uncompressed Length   000054EA 19F15 Filename Length       0018 19F17 Extra Length          0000 19F19 Comment Length        0000 19F1B Disk Start            0000 19F1D Int File Attributes   0000       [Bit 0]               0 'Binary Data' 19F1F Ext File Attributes   00000000 19F23 Local Header Offset   0000900F 19F27 Filename              'Mean.20160830_071500.zip'  19F3F CENTRAL HEADER #4     02014B50 19F43 Created Zip Spec      14 '2.0' 19F44 Created OS            00 'MS-DOS' 19F45 Extract Zip Spec      14 '2.0' 19F46 Extract OS            00 'MS-DOS' 19F47 General Purpose Flag  0808       [Bits 1-2]            0 'Normal Compression'       [Bit  3]              1 'Streamed'       [Bit 11]              1 'Language Encoding' 19F49 Compression Method    0008 'Deflated' 19F4B Last Mod Time         491E43F7 'Tue Aug 30 08:31:46 2016' 19F4F CRC                   C37437FB 19F53 Compressed Length     000073A0 19F57 Uncompressed Length   00008054 19F5B Filename Length       0018 19F5D Extra Length          0000 19F5F Comment Length        0000 19F61 Disk Start            0000 19F63 Int File Attributes   0000       [Bit 0]               0 'Binary Data' 19F65 Ext File Attributes   00000000 19F69 Local Header Offset   0000E06E 19F6D Filename              'Mean.20160830_080000.zip'  19F85 CENTRAL HEADER #5     02014B50 19F89 Created Zip Spec      14 '2.0' 19F8A Created OS            00 'MS-DOS' 19F8B Extract Zip Spec      14 '2.0' 19F8C Extract OS            00 'MS-DOS' 19F8D General Purpose Flag  0808       [Bits 1-2]            0 'Normal Compression'       [Bit  3]              1 'Streamed'       [Bit 11]              1 'Language Encoding' 19F8F Compression Method    0008 'Deflated' 19F91 Last Mod Time         491E43F7 'Tue Aug 30 08:31:46 2016' 19F95 CRC                   40E52180 19F99 Compressed Length     000049D3 19F9D Uncompressed Length   00005110 19FA1 Filename Length       0018 19FA3 Extra Length          0000 19FA5 Comment Length        0000 19FA7 Disk Start            0000 19FA9 Int File Attributes   0000       [Bit 0]               0 'Binary Data' 19FAB Ext File Attributes   00000000 19FAF Local Header Offset   00015454 19FB3 Filename              'Mean.20160830_070000.zip'  19FCB END CENTRAL HEADER    06054B50 19FCF Number of this disk   0000 19FD1 Central Dir Disk no   0000 19FD3 Entries in this disk  0005 19FD5 Total Entries         0005 19FD7 Size of Central Dir   0000015E 19FDB Offset to Central Dir 00019E6D 19FDF Comment Length        0000 Done 

Decompressed-recompressed file :

00000 LOCAL HEADER #1       04034B50 00004 Extract Zip Spec      14 '2.0' 00005 Extract OS            00 'MS-DOS' 00006 General Purpose Flag  0000 00008 Compression Method    0000 'Stored' 0000A Last Mod Time         491E510F 'Tue Aug 30 10:08:30 2016' 0000E CRC                   218B9162 00012 Compressed Length     00005595 00016 Uncompressed Length   00005595 0001A Filename Length       0018 0001C Extra Length          0000 0001E Filename              'Mean.20160830_070000.zip' 00036 PAYLOAD  055CB LOCAL HEADER #2       04034B50 055CF Extract Zip Spec      14 '2.0' 055D0 Extract OS            00 'MS-DOS' 055D1 General Purpose Flag  0000 055D3 Compression Method    0000 'Stored' 055D5 Last Mod Time         491E5117 'Tue Aug 30 10:08:46 2016' 055D9 CRC                   180124FD 055DD Compressed Length     00005972 055E1 Uncompressed Length   00005972 055E5 Filename Length       0018 055E7 Extra Length          0000 055E9 Filename              'Mean.20160830_071500.zip' 05601 PAYLOAD  0AF73 LOCAL HEADER #3       04034B50 0AF77 Extract Zip Spec      14 '2.0' 0AF78 Extract OS            00 'MS-DOS' 0AF79 General Purpose Flag  0000 0AF7B Compression Method    0000 'Stored' 0AF7D Last Mod Time         491E511D 'Tue Aug 30 10:08:58 2016' 0AF81 CRC                   03A4486C 0AF85 Compressed Length     00005953 0AF89 Uncompressed Length   00005953 0AF8D Filename Length       0018 0AF8F Extra Length          0000 0AF91 Filename              'Mean.20160830_073000.zip' 0AFA9 PAYLOAD  108FC LOCAL HEADER #4       04034B50 10900 Extract Zip Spec      14 '2.0' 10901 Extract OS            00 'MS-DOS' 10902 General Purpose Flag  0000 10904 Compression Method    0000 'Stored' 10906 Last Mod Time         491E5124 'Tue Aug 30 10:09:08 2016' 1090A CRC                   FEE97172 1090E Compressed Length     00008818 10912 Uncompressed Length   00008818 10916 Filename Length       0018 10918 Extra Length          0000 1091A Filename              'Mean.20160830_080000.zip' 10932 PAYLOAD  1914A LOCAL HEADER #5       04034B50 1914E Extract Zip Spec      14 '2.0' 1914F Extract OS            00 'MS-DOS' 19150 General Purpose Flag  0000 19152 Compression Method    0000 'Stored' 19154 Last Mod Time         491E5129 'Tue Aug 30 10:09:18 2016' 19158 CRC                   0B38337E 1915C Compressed Length     00004713 19160 Uncompressed Length   00004713 19164 Filename Length       0018 19166 Extra Length          0000 19168 Filename              'Mean.20160830_081500.zip' 19180 PAYLOAD  1D893 CENTRAL HEADER #1     02014B50 1D897 Created Zip Spec      14 '2.0' 1D898 Created OS            00 'MS-DOS' 1D899 Extract Zip Spec      14 '2.0' 1D89A Extract OS            00 'MS-DOS' 1D89B General Purpose Flag  0000 1D89D Compression Method    0000 'Stored' 1D89F Last Mod Time         491E510F 'Tue Aug 30 10:08:30 2016' 1D8A3 CRC                   218B9162 1D8A7 Compressed Length     00005595 1D8AB Uncompressed Length   00005595 1D8AF Filename Length       0018 1D8B1 Extra Length          0000 1D8B3 Comment Length        0000 1D8B5 Disk Start            0000 1D8B7 Int File Attributes   0000   [Bit 0]               0 'Binary Data' 1D8B9 Ext File Attributes   00000020   [Bit 5]               Archive 1D8BD Local Header Offset   00000000 1D8C1 Filename              'Mean.20160830_070000.zip'  1D8D9 CENTRAL HEADER #2     02014B50 1D8DD Created Zip Spec      14 '2.0' 1D8DE Created OS            00 'MS-DOS' 1D8DF Extract Zip Spec      14 '2.0' 1D8E0 Extract OS            00 'MS-DOS' 1D8E1 General Purpose Flag  0000 1D8E3 Compression Method    0000 'Stored' 1D8E5 Last Mod Time         491E5117 'Tue Aug 30 10:08:46 2016' 1D8E9 CRC                   180124FD 1D8ED Compressed Length     00005972 1D8F1 Uncompressed Length   00005972 1D8F5 Filename Length       0018 1D8F7 Extra Length          0000 1D8F9 Comment Length        0000 1D8FB Disk Start            0000 1D8FD Int File Attributes   0000   [Bit 0]               0 'Binary Data' 1D8FF Ext File Attributes   00000020   [Bit 5]               Archive 1D903 Local Header Offset   000055CB 1D907 Filename              'Mean.20160830_071500.zip'  1D91F CENTRAL HEADER #3     02014B50 1D923 Created Zip Spec      14 '2.0' 1D924 Created OS            00 'MS-DOS' 1D925 Extract Zip Spec      14 '2.0' 1D926 Extract OS            00 'MS-DOS' 1D927 General Purpose Flag  0000 1D929 Compression Method    0000 'Stored' 1D92B Last Mod Time         491E511D 'Tue Aug 30 10:08:58 2016' 1D92F CRC                   03A4486C 1D933 Compressed Length     00005953 1D937 Uncompressed Length   00005953 1D93B Filename Length       0018 1D93D Extra Length          0000 1D93F Comment Length        0000 1D941 Disk Start            0000 1D943 Int File Attributes   0000   [Bit 0]               0 'Binary Data' 1D945 Ext File Attributes   00000020   [Bit 5]               Archive 1D949 Local Header Offset   0000AF73 1D94D Filename              'Mean.20160830_073000.zip'  1D965 CENTRAL HEADER #4     02014B50 1D969 Created Zip Spec      14 '2.0' 1D96A Created OS            00 'MS-DOS' 1D96B Extract Zip Spec      14 '2.0' 1D96C Extract OS            00 'MS-DOS' 1D96D General Purpose Flag  0000 1D96F Compression Method    0000 'Stored' 1D971 Last Mod Time         491E5124 'Tue Aug 30 10:09:08 2016' 1D975 CRC                   FEE97172 1D979 Compressed Length     00008818 1D97D Uncompressed Length   00008818 1D981 Filename Length       0018 1D983 Extra Length          0000 1D985 Comment Length        0000 1D987 Disk Start            0000 1D989 Int File Attributes   0000   [Bit 0]               0 'Binary Data' 1D98B Ext File Attributes   00000020   [Bit 5]               Archive 1D98F Local Header Offset   000108FC 1D993 Filename              'Mean.20160830_080000.zip'  1D9AB CENTRAL HEADER #5     02014B50 1D9AF Created Zip Spec      14 '2.0' 1D9B0 Created OS            00 'MS-DOS' 1D9B1 Extract Zip Spec      14 '2.0' 1D9B2 Extract OS            00 'MS-DOS' 1D9B3 General Purpose Flag  0000 1D9B5 Compression Method    0000 'Stored' 1D9B7 Last Mod Time         491E5129 'Tue Aug 30 10:09:18 2016' 1D9BB CRC                   0B38337E 1D9BF Compressed Length     00004713 1D9C3 Uncompressed Length   00004713 1D9C7 Filename Length       0018 1D9C9 Extra Length          0000 1D9CB Comment Length        0000 1D9CD Disk Start            0000 1D9CF Int File Attributes   0000   [Bit 0]               0 'Binary Data' 1D9D1 Ext File Attributes   00000020   [Bit 5]               Archive 1D9D5 Local Header Offset   0001914A 1D9D9 Filename              'Mean.20160830_081500.zip'  1D9F1 END CENTRAL HEADER    06054B50 1D9F5 Number of this disk   0000 1D9F7 Central Dir Disk no   0000 1D9F9 Entries in this disk  0005 1D9FB Total Entries         0005 1D9FD Size of Central Dir   0000015E 1DA01 Offset to Central Dir 0001D893 1DA05 Comment Length        0000 Done 

Structure of the zip file

• MainFile.zip
  • InnerFile1.zip
    • InnerFile1.xml
  • InnerFile2.zip
    • InnerFile2.xml
  • InnerFile-N.zip
    • InnerFile-N.zip

Code to create zip file

public void addToZip(File zipFile, File... filesToAdd) {     final byte[] buffer = new byte[1024];      FileOutputStream fos = new FileOutputStream(zipFile.getAbsoluteFile());     ZipOutputStream zos = new ZipOutputStream(fos);      for (File fileToAdd : filesToAdd) {         ZipEntry entry = new ZipEntry(fileToAdd.getName());         try {             zos.putNextEntry(entry);             try (FileInputStream in = new FileInputStream(input.getAbsoluteFile())) {                 int len;                 while ((len = in.read(buffer)) > 0) {                     zos.write(buffer, 0, len);                 }             }             zos.closeEntry();         }     }     try {         zos.finish();         zos.close();     } catch (IOException ex) {         LOGGER.log(Level.SEVERE, null, ex);     } } 

Compression method

As per the primary reference for Zip files :

4.1.3 Data compression MAY be used to reduce the size of files placed into a ZIP file, but is not required. This format supports the use of multiple data compression algorithms. When compression is used, one of the documented compression algorithms MUST be used. Implementors are advised to experiment with their data to determine which of the available algorithms provides the best compression for their needs. Compression method 8 (Deflate) is the method used by default by most ZIP compatible application programs.

In this specific case, using zipEntry.setMethod(ZipEntry.DEFLATED); was not enough to have my files accepted by the legacy program.

2 Answers

Answers 1

I had (almost) the same problem with a banks legacy system. It checked the Compressed size and Uncompressed size in the archive. In your provided output the accepted one contains valid informations (diff):

00012 Compressed Length     00005595 00012 Compressed Length     00005595 

Those fields in your Java generated archive file:

00012 Compressed Length     00000000 00016 Uncompressed Length   00000000 

For more detailed header information checkout Wikipedia.

Most likely the soulution to your problem is this:

http://stackoverflow.com/a/29096370/4823977

Answers 2

I hate guessing without being able to verify my assumptions, thus the comment/question about the availability of the actual server-side file checker on the FTP server, but for what it is worth and just assuming that the checker prefers STORED entries to DEFLATED ones, here is how you can create

• deflated ZIP files from XML files (which makes sense because XML is nicely compressible) and
• stored ZIP files from nested ZIP files (which also makes sense because usually compressed files cannot effectively get smaller when re-compressing them).

The problem here is that when using STORED entries it is mandatory to also set (un-)compressed size and CRC for each entry. This is how you can do it (unfortunately I am reading each nested ZIP file twice, once for calculating the CRC and once for copying its contents to the ZIP output stream):

Driver application:

package de.scrum_master.app;  import java.io.File; import java.io.IOException; import java.util.SortedMap; import java.util.TreeMap;  public class Application {     public static void main(String[] args) throws IOException {         SortedMap<File, File[]> fileMappings = new TreeMap<>();         fileMappings.put(             new File("books.zip"),             new File[] { new File("books.xml") }         );         fileMappings.put(             new File("cds.zip"),             new File[] { new File("cds.xml") }         );         fileMappings.put(             new File("clothes.zip"),             new File[] { new File("clothes.xml") }         );          ZipTool zipTool = new ZipTool();         for (File zipFile : fileMappings.keySet())             zipTool.addToZip(zipFile, fileMappings.get(zipFile));         zipTool.addToZip(             new File("archive.zip"),             new File[] { new File("books.zip"), new File("cds.zip"), new File("clothes.zip") }         );     } } 

ZIP utility:

package de.scrum_master.app;  import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.util.zip.CRC32; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream;  public class ZipTool {     public void addToZip(File zipFile, File... filesToAdd) throws IOException {         try (             FileOutputStream fos = new FileOutputStream(zipFile.getAbsoluteFile());             ZipOutputStream zos = new ZipOutputStream(fos)         ) {             final byte[] buffer = new byte[1024];             for (File fileToAdd : filesToAdd) {                 ZipEntry entry = new ZipEntry(fileToAdd.getName());                 if (fileToAdd.getName().endsWith(".zip")) {                     entry.setMethod(ZipEntry.STORED);                     long fileSize = fileToAdd.length();                     entry.setSize(fileSize);                     entry.setCompressedSize(fileSize);                     CRC32 crc32 = new CRC32();                     try (FileInputStream in = new FileInputStream(fileToAdd.getAbsoluteFile())) {                         int len;                         while ((len = in.read(buffer)) > 0) {                             crc32.update(buffer, 0, len);                         }                     }                     entry.setCrc(crc32.getValue());                 }                 zos.putNextEntry(entry);                 try (FileInputStream in = new FileInputStream(fileToAdd.getAbsoluteFile())) {                     int len;                     while ((len = in.read(buffer)) > 0) {                         zos.write(buffer, 0, len);                     }                 }                 zos.closeEntry();             }         }     } } 

Maybe you can try that and provide me with feedback. If this is not good enough and the checker still complains, we can try something else to make the zip archives even more similar.

Read More

Rails upload to AWS creating .zip.cpgz file loop

I'm running into a weird situation where some files, specifically ZIP format, when uploaded to AWS within my Rails app are corrupted/converted. When downloaded and decompressed they turn into a CPGZ format, which decompresses back into a ZIP, and infinitely does this.

I haven't noticed a pattern that causes this, so it's seemingly sporadic, and can confirm that the files are not corrupt before upload. The only other issue/topic I've found on this relates to PHP, and seems to be different circumstances.

I am using AWS SDK for Ruby v1 (not v2 because of my Rails version) and jQuery-File-Upload. Since some of the files are large, I am using chunked uploads.

In my controller, the presigned POST URL is created like this:

S3_BUCKET.presigned_post(key: "uploads/#{SecureRandom.uuid}-${filename}", success_action_status: '201') 

And jQuery File Upload is set up like so (some parts removed for brevity):

this.$el.fileupload({   fileInput: this.uploadField, // this is an <input type="file">   url: this.awsURL, // https://BUCKET.s3.amazonaws.com/   formData: JSON.parse(this.awsData), // {"AWSAccessKeyId":"...","key":"uploads/1234-${filename}","policy":"...","signature":"...","success_action_status":"201"}   type: 'POST',   autoUpload: true,   paramName: 'file',   dataType: 'XML',   replaceFileInput: false,   maxChunkSize: 1000000,   add: function(event, data) {     var file = data.files[0];     var fileType = file.type;      // Check file type     if (~'ai sketch psd jpg jpeg png zip ttf woff eot gif'.indexOf(fileType.toLowerCase())) {       return alert('Sorry, that file type is not supported');     };      data.submit();   },   progress: function(event, data) {     // Display progress   },   done: function(event, data) {     var file = data.files[0];     var fileName = file.name.replace(/ /g,"_");     var item = _this.uploadedItems[fileName];     var key = $(data.jqXHR.responseXML).find("Key").text();     // awsHost = BUCKET.s3.amazonaws.com     var url = '//' + _this.awsHost + '/' + key;      // Set form values using above info   },   fail: function(event, data) {     // Alert failure   } }); 

Has anyone experienced this? It's very frustrating.

1 Answers

Answers 1

Set content-type to application/zip when you sending the request.

SEE https://github.com/aws/aws-sdk-ruby/blob/aws-sdk-v1/lib/aws/s3/presigned_post.rb

Read More