package com.test;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.Random;
public class LargeMappedFiles {
public static void main(String[] args) throws IOException {
//The path to write to the file & NBSP;
String filePath = "D:\456";
//The path of the sharded file & NBSP;
String sqlitFilePath = "D:\456\123";
//Number of data & NBSP;
int CountNumbers=10000000;
//Number of subfiles & NBSP;
int CountFile=10;
//Precision
int countAccuracy=30*CountFile;
long startNumber=System.currentTimeMillis();
//Write to big data file & NBSP;
WriteData(filePath,CountNumbers);
System.out.println(" Are stored ");
//Divide the big data file into ten other small files & NBSP;
sqlitFileDate(filePath, sqlitFilePath,CountFile);
System.out.println(" File cut! ");
//Sort the data for each file & NBSP;
singleFileDataSort(sqlitFilePath,CountFile);
System.out.println(" Sorted for each subfile! ");
//Precision adjustment, comparison and integration of ten file data & PI;
deathDataFile(filePath,sqlitFilePath,countAccuracy,CountFile);
System.out.println(" Integration to complete ");
long stopNumber=System.currentTimeMillis();
System.out.println(" Time consuming "+(stopNumber-startNumber)/1000+" ms ");
}
//Write to big data file & NBSP;
public static void WriteData(String path,int CountNumbers) throws IOException {
path = path + "\12114.txt";
FileWriter fs = new FileWriter(path);
BufferedWriter fw=new BufferedWriter(fs);
for (int i = 0; i < CountNumbers; i++) {
fw.write(new Random().nextInt(Integer.MAX_VALUE) + "rn");
}
fw.close();
fs.close();
}
//Divide the big data file into ten other small files & NBSP;
public static void sqlitFileDate(String filepath, String sqlitPath,
int CountFile) throws IOException {
FileWriter fs = null;
BufferedWriter fw=null;
FileReader fr = new FileReader(filepath + "\12114.txt");
BufferedReader br = new BufferedReader(fr); //Reads gets the entire row of data & NBSP;
int i = 1;
LinkedList WriterLists=new LinkedList(); //Initializes a collection of file flow objects & NBSP;
LinkedList fwLists=new LinkedList();
for (int j = 1; j <= CountFile; j++) {
//Declaration object & NBSP;
fs = new FileWriter(sqlitPath + "\12" + j + ".txt",false);
fw=new BufferedWriter(fs);
//Loads an object into a collection & NBSP;
WriterLists.add(fs);
fwLists.add(fw);
}
//Determines whether there is any data returned in the file stream & NBSP;
while (br.ready()) {
int count=1;//Initializes the first file stream & NBSP;
for (Iterator iterator = fwLists.iterator(); iterator.hasNext();) {
BufferedWriter type = (BufferedWriter) iterator.next();
if(i==count)//Determine which file stream is writing the data & NBSP;
{
//Write data, jump, proceed to the next file stream, write the next data & NBSP;
type.write(br.readLine() + "rn");
break;
}
count++;
}
//Determining if the last file flow is reached & NBSP;
if (i >= CountFile) {
i = 1;
} else
i++;
}
br.close();
fr.close();
for (Iterator iterator = fwLists.iterator(); iterator.hasNext();) {
BufferedWriter object = (BufferedWriter) iterator.next();
object.close();
}
//Traversing closes all subfile flows & NBSP;
for (Iterator iterator = WriterLists.iterator(); iterator.hasNext();) {
FileWriter object = (FileWriter) iterator.next();
object.close();
}
}
//Sort the data for each file & NBSP;
public static void singleFileDataSort(String path1,int CountFile) throws IOException {
LinkedList nums = null;
for (int i = 1; i <= CountFile; i++) {
nums = new LinkedList();
String path = path1 + "\12" + i + ".txt";
try {
FileReader fr = new FileReader(path);
BufferedReader br = new BufferedReader(fr);
while (br.ready()) {
//Adds a single read to the collection & NBSP;
nums.add(Integer.parseInt(br.readLine()));
}
//Sorting a collection & NBSP;
Collections.sort(nums);
//Writes sorted data to a source file & NBSP;
numberSort(nums, path);
br.close();
fr.close();
} catch (NumberFormatException e) {
e.printStackTrace();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
}
//Sort the data for each file, then write to the source file & NBSP;
public static void numberSort(LinkedList list, String path) {
try {
FileWriter fs = new FileWriter(path);
BufferedWriter fw=new BufferedWriter(fs);
for (Iterator iterator = list.iterator(); iterator.hasNext();) {
Object object = (Object) iterator.next();
fw.write(object + "rn");
}
fw.close();
fs.close();
} catch (IOException e) {
e.printStackTrace();
}
}
//Final integration of file data (precision adjustment)& NBSP;
public static void deathDataFile(String filepath, String sqlitFilePath1,
int countAccuracy, int CountFile) throws IOException {
LinkedList nums = new LinkedList(); //Add data, sort & NBSP;
Object temp = null; //Records the last number left in each sort & NBSP;
boolean ispass = false;
LinkedList ispasses = null; //Recording the status information of the data file & NBSP;
FileWriter fs = new FileWriter(filepath + "\Sort.txt", false); //Create a file stream so that consolidated data is written & NBSP;
BufferedWriter bw=new BufferedWriter(fs);
FileReader fr = null; //Declaration read file stream & NBSP;
BufferedReader br = null; //The statement BufferedReader
LinkedList WriterLists = new LinkedList(); // Initializes a collection of file flow objects & NBSP;
LinkedList WriterListFile = new LinkedList();
for (int j = 1; j <= CountFile; j++) {
//Declare an object that opens all subfile streams to access the data & NBSP; of all subfiles.
fr = new FileReader(sqlitFilePath1 + "\12" + j + ".txt");
//Open all bufferedreaders for the next full line reading & NBSP;
br = new BufferedReader(fr);
//Load all the FileReader objects into the collection & NBSP;
WriterListFile.add(fr);
//Loads all BufferedReader objects into the collection & NBSP;
WriterLists.add(br);
}
for (;;) {
//Store the presence or absence of data from ten source files in the collection for later judgment.
ispasses = new LinkedList();
//Reads a single data from each of the ten source files & NBSP;
for (Iterator iterator = WriterLists.iterator(); iterator.hasNext();) {
BufferedReader object = (BufferedReader) iterator.next();
Object obj = null;
while (object.ready()) {
//Add data every time for all file streams & NBSP;
nums.add(Integer.parseInt(object.readLine().toString()));
break;
}
if (object.ready() == false)
ispasses.add("true"); //Storing the state of the data in each file in a collection & NBSP;
}
//Is the decision the first time in & NBSP;
if (nums.size() % countAccuracy == 0 && ispass == false) {
//Sorting a collection & NBSP;
Collections.sort(nums);
//Receives the largest data, and writes the rest to the general sort file & NBSP;
temp = numberSortData(nums, filepath, false, countAccuracy, bw);
//Re-initialize the set & NBSP;
nums = new LinkedList();
//Add the previous set to compare the rest of the data & NBSP;
nums.add(temp);
ispass = true;
//Record the amount of data in the source file for the next iteration.
continue;
}
if (ispass) {
if (nums.size() % countAccuracy == 1 && nums.size() > 1) {
//Sorting a collection & NBSP;
Collections.sort(nums);
//Receives the largest data, and writes the rest to the general sort file & NBSP;
temp = numberSortData(nums, filepath, true, countAccuracy,
bw);
nums = new LinkedList();
nums.add(temp);
continue;
}
}
//Record the location of a set of data
//No data to determine if there are ten files & NBSP;
if (ispasses.size() == CountFile) {
Collections.sort(nums);
temp = numberSortData(nums, filepath, true, countAccuracy, bw);
nums = new LinkedList();
break;
}
}
bw.close();
//Close write entry & NBSP;
fs.close();
//Close all BufferedReader ;
for (Iterator iterator = WriterLists.iterator(); iterator.hasNext();) {
BufferedReader object2 = (BufferedReader) iterator.next();
object2.close();
}
//Close all FileReader
for (Iterator iterator = WriterListFile.iterator(); iterator.hasNext();) {
FileReader object = (FileReader) iterator.next();
object.close();
}
}
//Sort the data and write it to the final file (precision adjustment)& NBSP;
public static Object numberSortData(LinkedList list, String filePath,
boolean ispass, int countAccuracy,BufferedWriter fs) {
Object temp = 0; //Record the last value & NBSP;
int tempCount = 0; //The data location to which the record is written & NBSP;
try {
for (Iterator iterator = list.iterator(); iterator.hasNext();) {
Object object = (Object) iterator.next();
//Judge if it is the last number & NBSP;
if (tempCount == list.size() - 1) {
//There is not a hundred tricks in the collection of judgments;
if (list.size() < countAccuracy + 1 && ispass) {
temp = null;
} else {
temp = object;
break;
}
}
//Write to data source & NBSP;
fs.write(object + "rn");
//The index & NBSP; of the recorded data;
tempCount++;
}
} catch (IOException e) {
e.printStackTrace();
}
return temp;
}
}