Skip to content

Instantly share code, notes, and snippets.

@youthlin
Created October 21, 2017 06:43
Show Gist options
  • Save youthlin/c903f52dd381875638b5457e08984e66 to your computer and use it in GitHub Desktop.
Save youthlin/c903f52dd381875638b5457e08984e66 to your computer and use it in GitHub Desktop.
import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.Scanner;
public class Downloader {
private static final String urlListFile = "out.txt";
public static void main(String[] args) throws Exception {
Scanner in = new Scanner(new File(urlListFile));
List<String> urlList = new ArrayList<String>();
while (in.hasNext()) {
String url = in.nextLine();
urlList.add(url);
}
int total = urlList.size();
System.out.println(total);
int i = 0;
for (String url : urlList) {
i++;
download(url);
System.out.println(i + "/" + total);
Thread.sleep(1000);
}
}
private static boolean download(String urlListFile) throws IOException {
// http://youthlin.qiniudn.com/wp-content/uploads/2017/05/NewPost.png
byte[] imageFromNetByUrl = getImageFromNetByUrl(urlListFile);
writeImageToDisk(imageFromNetByUrl, urlListFile.substring("http://youthlin.qiniudn.com/".length()));
return true;
}
private static byte[] getImageFromNetByUrl(String strUrl) {
try {
URL url = new URL(strUrl);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("GET");
conn.setConnectTimeout(5 * 1000);
InputStream inStream = conn.getInputStream();// 通过输入流获取图片数据
return readInputStream(inStream);
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
private static byte[] readInputStream(InputStream inStream) throws Exception {
ByteArrayOutputStream outStream = new ByteArrayOutputStream();
byte[] buffer = new byte[1024];
int len = 0;
while ((len = inStream.read(buffer)) != -1) {
outStream.write(buffer, 0, len);
}
inStream.close();
return outStream.toByteArray();
}
private static void writeImageToDisk(byte[] img, String fileName) {
try {
File file = new File(fileName);
File parentFile = file.getParentFile();
if (!parentFile.exists()) {
parentFile.mkdirs();
}
if (!file.exists()) {
file.createNewFile();
}
FileOutputStream fops = new FileOutputStream(file);
fops.write(img);
fops.flush();
fops.close();
System.out.println("downloaded: " + file.getAbsolutePath());
} catch (Exception e) {
e.printStackTrace();
}
}
}
import com.qiniu.common.Zone;
import com.qiniu.storage.BucketManager;
import com.qiniu.storage.Configuration;
import com.qiniu.storage.model.FileInfo;
import com.qiniu.util.Auth;
import java.io.*;
import java.util.ArrayList;
import java.util.List;
public class Task {
private static final String accessKey = "xxx";
private static final String secretKey = "xxx";
private static final String bucket = "xxx";
public static final String domain = "http://xxx.qiniudn.com/";
public static void main(String[] args) throws IOException {
//构造一个带指定Zone对象的配置类
Configuration cfg = new Configuration(Zone.zone0());
//...其他参数参考类注释
Auth auth = Auth.create(accessKey, secretKey);
BucketManager bucketManager = new BucketManager(auth, cfg);
//文件名前缀
String prefix = "wp-content/uploads";
//每次迭代的长度限制,最大1000,推荐值 1000
int limit = 1000;
//指定目录分隔符,列出所有公共前缀(模拟列出目录效果)。缺省值为空字符串
String delimiter = "";
//列举空间文件列表
BucketManager.FileListIterator fileListIterator =
bucketManager.createFileListIterator(bucket, prefix, limit, delimiter);
List<String> nameList = new ArrayList<String>();
while (fileListIterator.hasNext()) {
//处理获取的file list结果
FileInfo[] items = fileListIterator.next();
if (items == null) {
continue;
}
for (FileInfo item : items) {
System.out.println(item.key);
System.out.println(item.hash);
System.out.println(item.fsize);
System.out.println(item.mimeType);
System.out.println(item.putTime);
System.out.println(item.endUser);
System.out.println("------------");
nameList.add(domain + item.key);
}
System.out.println("total: " + items.length);
}
File file = new File("out.txt");
PrintWriter out = new PrintWriter(file);
for (String name : nameList) {
out.println(name);
}
out.flush();
out.close();
System.out.println(file.getAbsolutePath());
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment