Node.js has revolutionized server-side development, and when it comes to handling file uploads and streaming, it really shines. Let’s dive into creating efficient file upload and streaming servers using Node.js and Multer.
First things first, we need to set up our project. Open your terminal and create a new directory for your project. Navigate into it and initialize a new Node.js project:
mkdir file-upload-server
cd file-upload-server
npm init -y
Now, let’s install the necessary dependencies:
npm install express multer
Express is our web framework, and Multer is a middleware for handling multipart/form-data, which is primarily used for uploading files.
Let’s create our server file. Create a new file called server.js and add the following code:
const express = require('express');
const multer = require('multer');
const path = require('path');
const fs = require('fs');
const app = express();
const port = 3000;
// Configure storage
const storage = multer.diskStorage({
destination: (req, file, cb) => {
cb(null, 'uploads/');
},
filename: (req, file, cb) => {
cb(null, Date.now() + path.extname(file.originalname));
}
});
const upload = multer({ storage: storage });
// Serve the HTML form
app.get('/', (req, res) => {
res.sendFile(__dirname + '/index.html');
});
// Handle file upload
app.post('/upload', upload.single('file'), (req, res) => {
if (!req.file) {
return res.status(400).send('No file uploaded.');
}
res.send('File uploaded successfully!');
});
app.listen(port, () => {
console.log(`Server running at http://localhost:${port}`);
});
This code sets up a basic Express server with a route for serving an HTML form and another route for handling file uploads. We’re using Multer’s diskStorage engine to customize where files are stored and how they’re named.
Now, let’s create a simple HTML form for uploading files. Create a file named index.html in the same directory:
<!DOCTYPE html>
<html>
<head>
<title>File Upload</title>
</head>
<body>
<h1>Upload a File</h1>
<form action="/upload" method="POST" enctype="multipart/form-data">
<input type="file" name="file">
<input type="submit" value="Upload">
</form>
</body>
</html>
Great! We now have a basic file upload server. But what about handling large files? That’s where streaming comes in handy.
Let’s modify our server to handle large file uploads using streams. Update your server.js file:
const express = require('express');
const multer = require('multer');
const path = require('path');
const fs = require('fs');
const app = express();
const port = 3000;
// Configure storage
const storage = multer.memoryStorage();
const upload = multer({ storage: storage });
// Serve the HTML form
app.get('/', (req, res) => {
res.sendFile(__dirname + '/index.html');
});
// Handle file upload
app.post('/upload', upload.single('file'), (req, res) => {
if (!req.file) {
return res.status(400).send('No file uploaded.');
}
const fileName = Date.now() + path.extname(req.file.originalname);
const writeStream = fs.createWriteStream(`uploads/${fileName}`);
writeStream.write(req.file.buffer);
writeStream.end();
writeStream.on('finish', () => {
res.send('File uploaded successfully!');
});
writeStream.on('error', (err) => {
console.error(err);
res.status(500).send('An error occurred during file upload.');
});
});
app.listen(port, () => {
console.log(`Server running at http://localhost:${port}`);
});
In this updated version, we’re using memoryStorage() instead of diskStorage(). This stores the file in memory as a Buffer. We then use a write stream to write the file to disk chunk by chunk, which is more efficient for larger files.
Now, let’s take it a step further and add progress reporting for our file uploads. We’ll use the busboy library for this, as it gives us more control over the upload process. First, install busboy:
npm install busboy
Now, update your server.js file:
const express = require('express');
const busboy = require('busboy');
const path = require('path');
const fs = require('fs');
const app = express();
const port = 3000;
// Serve the HTML form
app.get('/', (req, res) => {
res.sendFile(__dirname + '/index.html');
});
// Handle file upload
app.post('/upload', (req, res) => {
const bb = busboy({ headers: req.headers });
let fileName = '';
let fileSize = 0;
let uploadedSize = 0;
bb.on('file', (name, file, info) => {
fileName = Date.now() + path.extname(info.filename);
const writeStream = fs.createWriteStream(`uploads/${fileName}`);
file.on('data', (data) => {
uploadedSize += data.length;
const progress = (uploadedSize / fileSize) * 100;
console.log(`Progress: ${progress.toFixed(2)}%`);
});
file.pipe(writeStream);
});
bb.on('field', (name, val, info) => {
if (name === 'fileSize') {
fileSize = parseInt(val);
}
});
bb.on('finish', () => {
res.send('File uploaded successfully!');
});
req.pipe(bb);
});
app.listen(port, () => {
console.log(`Server running at http://localhost:${port}`);
});
This version uses busboy to handle the file upload. It reports progress to the console, but you could easily modify this to send progress updates to the client using WebSockets or Server-Sent Events.
To make this work, we need to modify our HTML form to include the file size. Update your index.html:
<!DOCTYPE html>
<html>
<head>
<title>File Upload</title>
</head>
<body>
<h1>Upload a File</h1>
<form action="/upload" method="POST" enctype="multipart/form-data">
<input type="file" name="file" id="file">
<input type="hidden" name="fileSize" id="fileSize">
<input type="submit" value="Upload">
</form>
<script>
document.getElementById('file').addEventListener('change', function(e) {
document.getElementById('fileSize').value = this.files[0].size;
});
</script>
</body>
</html>
This form now includes a hidden input field that stores the file size, which is set using JavaScript when a file is selected.
Now we have a robust file upload system that can handle large files efficiently and report progress. But what about streaming? Let’s add a route for streaming video files.
Add this to your server.js:
// Stream video
app.get('/video/:filename', (req, res) => {
const filename = req.params.filename;
const filePath = `uploads/${filename}`;
const stat = fs.statSync(filePath);
const fileSize = stat.size;
const range = req.headers.range;
if (range) {
const parts = range.replace(/bytes=/, "").split("-");
const start = parseInt(parts[0], 10);
const end = parts[1] ? parseInt(parts[1], 10) : fileSize-1;
const chunksize = (end-start)+1;
const file = fs.createReadStream(filePath, {start, end});
const head = {
'Content-Range': `bytes ${start}-${end}/${fileSize}`,
'Accept-Ranges': 'bytes',
'Content-Length': chunksize,
'Content-Type': 'video/mp4',
};
res.writeHead(206, head);
file.pipe(res);
} else {
const head = {
'Content-Length': fileSize,
'Content-Type': 'video/mp4',
};
res.writeHead(200, head);
fs.createReadStream(filePath).pipe(res);
}
});
This route handles video streaming. It supports range requests, which allows clients to request specific parts of the video file. This is crucial for features like seeking in video players.
To test this, you can upload a video file using your form, then access it via the /video/:filename route. You could add a simple video player to your HTML to test it out:
<video controls>
<source src="/video/your-video-filename.mp4" type="video/mp4">
Your browser does not support the video tag.
</video>
Remember to replace ‘your-video-filename.mp4’ with the actual filename of your uploaded video.
Now we’ve covered file uploads, progress reporting, and video streaming. But what about security? It’s crucial to implement proper security measures when dealing with file uploads.
Here are a few security considerations:
-
Limit file size: You can use Multer’s
limitsoption to restrict file size. -
Validate file types: Only allow specific file types to be uploaded.
-
Scan for malware: Implement virus scanning for uploaded files.
-
Use secure file names: Don’t use user-provided filenames directly.
-
Store files outside the web root: This prevents direct access to uploaded files.
Let’s implement some of these. Update your server.js:
const express = require('express');
const busboy = require('busboy');
const path = require('path');
const fs = require('fs');
const crypto = require('crypto');
const app = express();
const port = 3000;
const UPLOAD_PATH = path.join(__dirname, 'secure_uploads');
const MAX_FILE_SIZE = 50 * 1024 * 1024; // 50MB
const ALLOWED_FILE_TYPES = ['image/jpeg', 'image/png', 'video/mp4'];
// Create upload directory if it doesn't exist
if (!fs.existsSync(UPLOAD_PATH)) {
fs.mkdirSync(UPLOAD_PATH);
}
app.get('/', (req, res) => {
res.sendFile(__dirname + '/index.html');
});
app.post('/upload', (req, res) => {
const bb = busboy({
headers: req.headers,
limits: { fileSize: MAX_FILE_SIZE }
});
let saveFile = false;
let fileStream;
bb.on('file', (name, file, info) => {
if (!ALLOWED_FILE_TYPES.includes(info.mimeType)) {
return res.status(400).send('Invalid file type');
}
const fileExt = path.extname(info.filename);
const newFilename = crypto.randomBytes(16).toString('hex') + fileExt;
const filePath = path.join(UPLOAD_PATH, newFilename);
fileStream = fs.createWriteStream(filePath);
saveFile = true;
file.pipe(fileStream);
file.on('limit', () => {
saveFile = false;
fs.unlink(filePath, () => {});
res.status(400).send('File too large');
});
});
bb.on('finish', () => {
if (saveFile) {
res.send('File uploaded successfully!');
}
});
req.pipe(bb);
});
app.listen(port, () => {