About SpringBoot Large File RestTemplate Download Solution
- 2021-12-04 10:04:41
- OfStack
Recently, based on the RestTemplate downloading file stream used in the project, it takes 3-4 minutes to download large files above 1G, because there is no fragmentation and multithreading when calling API interface, and all file streams are loaded synchronously, which makes the performance slow. Recently, combined with online cases and my own summary, I wrote a project of downloading tuling/fileServer in pieces: 1. Loading and outputting related codes in the browser including synchronous download file stream; 2. Contains the relevant codes for downloading fragmented files and merging files by fragmented multithreading;
In addition, in the DownloadThread project, the code is used to complete a remote RestUrl request to obtain a large file of remote resources for multi-thread fragmentation and download to a local case, and a large video file such as. mp4/. avi can be downloaded. Related codes are also 1 and packaged and uploaded.
Synchronous download, support fragment download Range main code:
@Controller
public class DownLoadController {
private static final String UTF8 = "UTF-8";
@RequestMapping("/download")
public void downLoadFile(HttpServletRequest request, HttpServletResponse response) throws IOException {
File file = new File("D:\\DevTools\\ideaIU-2021.1.3.exe");
response.setCharacterEncoding(UTF8);
InputStream is = null;
OutputStream os = null;
try {
// Sliced download Range Representation bytes=100-1000 100-
long fSize = file.length();
response.setContentType("application/x-download");
String fileName = URLEncoder.encode(file.getName(), UTF8);
response.addHeader("Content-Disposition", "attachment;filename=" + fileName);
// Support fragmented download
response.setHeader("Accept-Range", "bytes");
response.setHeader("fSize", String.valueOf(fSize));
response.setHeader("fName", fileName);
long pos = 0, last = fSize - 1, sum = 0;
if (null != request.getHeader("Range")) {
response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
String numberRange = request.getHeader("Range").replaceAll("bytes=", "");
String[] strRange = numberRange.split("-");
if (strRange.length == 2) {
pos = Long.parseLong(strRange[0].trim());
last = Long.parseLong(strRange[1].trim());
if (last > fSize-1) {
last = fSize - 1;
}
} else {
pos = Long.parseLong(numberRange.replaceAll("-", "").trim());
}
}
long rangeLength = last - pos + 1;
String contentRange = new StringBuffer("bytes").append(pos).append("-").append(last).append("/").append(fSize).toString();
response.setHeader("Content-Range", contentRange);
response.setHeader("Content-Length", String.valueOf(rangeLength));
os = new BufferedOutputStream(response.getOutputStream());
is = new BufferedInputStream(new FileInputStream(file));
is.skip(pos);
byte[] buffer = new byte[1024];
int length = 0;
while (sum < rangeLength) {
int readLength = (int) (rangeLength - sum);
length = is.read(buffer, 0, (rangeLength - sum) <= buffer.length ? readLength : buffer.length);
sum += length;
os.write(buffer,0, length);
}
System.out.println(" Download complete ");
}finally {
if (is != null){
is.close();
}
if (os != null){
os.close();
}
}
}
}Multi-thread slicing downloads slicing files, and merges slicing main codes after downloading:
@RestController
public class DownloadClient {
private static final Logger LOGGER = LoggerFactory.getLogger(DownloadClient.class);
private final static long PER_PAGE = 1024L * 1024L * 50L;
private final static String DOWN_PATH = "F:\\fileItem";
ExecutorService taskExecutor = Executors.newFixedThreadPool(10);
@RequestMapping("/downloadFile")
public String downloadFile() {
// Probe download
FileInfo fileInfo = download(0, 10, -1, null);
if (fileInfo != null) {
long pages = fileInfo.fSize / PER_PAGE;
for (long i = 0; i <= pages; i++) {
Future<FileInfo> future = taskExecutor.submit(new DownloadThread(i * PER_PAGE, (i + 1) * PER_PAGE - 1, i, fileInfo.fName));
if (!future.isCancelled()) {
try {
fileInfo = future.get();
} catch (InterruptedException | ExecutionException e) {
e.printStackTrace();
}
}
}
return System.getProperty("user.home") + "\\Downloads\\" + fileInfo.fName;
}
return null;
}
class FileInfo {
long fSize;
String fName;
public FileInfo(long fSize, String fName) {
this.fSize = fSize;
this.fName = fName;
}
}
/**
* According to the starting position / End position
* Download files by slices, and temporarily store files by slices
* File size = End position - Start position
*
* @return
*/
private FileInfo download(long start, long end, long page, String fName) {
File dir = new File(DOWN_PATH);
if (!dir.exists()) {
dir.mkdirs();
}
// Breakpoint download
File file = new File(DOWN_PATH, page + "-" + fName);
if (file.exists() && page != -1 && file.length() == PER_PAGE) {
return null;
}
try {
HttpClient client = HttpClients.createDefault();
HttpGet httpGet = new HttpGet("http://127.0.0.1:8080/download");
httpGet.setHeader("Range", "bytes=" + start + "-" + end);
HttpResponse response = client.execute(httpGet);
String fSize = response.getFirstHeader("fSize").getValue();
fName = URLDecoder.decode(response.getFirstHeader("fName").getValue(), "UTF-8");
HttpEntity entity = response.getEntity();
InputStream is = entity.getContent();
FileOutputStream fos = new FileOutputStream(file);
byte[] buffer = new byte[1024];
int ch;
while ((ch = is.read(buffer)) != -1) {
fos.write(buffer, 0, ch);
}
is.close();
fos.flush();
fos.close();
// Finally 1 Pieces
if (end - Long.parseLong(fSize) > 0) {
// Start merging files
mergeFile(fName, page);
}
return new FileInfo(Long.parseLong(fSize), fName);
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
private void mergeFile(String fName, long page) {
File file = new File(DOWN_PATH, fName);
try {
BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(file));
for (long i = 0; i <= page; i++) {
File tempFile = new File(DOWN_PATH, i + "-" + fName);
while (!file.exists() || (i != page && tempFile.length() < PER_PAGE)) {
try {
Thread.sleep(100);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
byte[] bytes = FileUtils.readFileToByteArray(tempFile);
os.write(bytes);
os.flush();
tempFile.delete();
}
File testFile = new File(DOWN_PATH, -1 + "-null");
testFile.delete();
os.flush();
os.close();
} catch (IOException e) {
e.printStackTrace();
}
}
/**
* Get the remote file size
*/
private long getRemoteFileSize(String remoteFileUrl) throws IOException {
long fileSize = 0;
HttpURLConnection httpConnection = (HttpURLConnection) new URL(remoteFileUrl).openConnection();
// Use HEAD Method
httpConnection.setRequestMethod("HEAD");
int responseCode = httpConnection.getResponseCode();
if (responseCode >= 400) {
LOGGER.debug("Web Server response error !");
return 0;
}
String sHeader;
for (int i = 1;; i++) {
sHeader = httpConnection.getHeaderFieldKey(i);
if (sHeader != null && sHeader.equals("Content-Length")) {
LOGGER.debug(" File size ContentLength:" + httpConnection.getContentLength());
fileSize = Long.parseLong(httpConnection.getHeaderField(sHeader));
break;
}
}
return fileSize;
}
class DownloadThread implements Callable<FileInfo> {
long start;
long end;
long page;
String fName;
public DownloadThread(long start, long end, long page, String fName) {
this.start = start;
this.end = end;
this.page = page;
this.fName = fName;
}
@Override
public FileInfo call() {
return download(start, end, page, fName);
}
}
}The code is available in the local pro-test (repaired Bug), but what is lacking at present is that it does not realize the download display corresponding to the browser when downloading fragments, and needs to temporarily exist in the local disk directory. At present, the code will be open source, hoping to have a better solution of Coder Fork support! Star is also welcome to join us.
The blog post refers to the related case tutorial of Turing College, and modifies some code implementations:
WebUploader-based on SpringBoot, Java file upload and download high-level actual combat
The code for this article has been uploaded to GitHub:
BurstDownload