using System;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
using HtmlAgilityPack;
namespace DownloadBbc {
internal class Program {
private static void DownloadFile(string mp3) {
if (string.IsNullOrWhiteSpace(mp3))
return;
var fileName = Path.GetFileName(mp3);
if (File.Exists(fileName))
return;
using (var webClient = new WebClient()) {
webClient.DownloadFile(mp3, fileName);
}
}
private static async Task RunAsync(string url) {
using (var client = new HttpClient()) {
var response = await client.GetAsync(url);
if (response.IsSuccessStatusCode) {
var html = await response.Content.ReadAsStringAsync();
var page = new HtmlDocument {OptionAutoCloseOnEnd = false, OptionFixNestedTags = true};
page.LoadHtml(html);
var mp3s =
from mp3 in page.DocumentNode.Descendants("a")
where mp3.GetAttributeValue("href", "").EndsWith(".mp3")
select mp3.GetAttributeValue("href", "");
// Read the file and display it line by line.
var file = new StreamReader("bbc.txt");
while ((line = file.ReadLine()) != null) {
Console.WriteLine(line);
RunAsync(line).Wait();
}
matches := regularExpression.FindAllString(htmlContents, -1)
for _, mp3Url := range matches {
fmt.Println(mp3Url)
var mp3FileName = getFileName(mp3Url)
if _, err := os.Stat(mp3FileName); os.IsNotExist(err) {
//fmt.Printf("no such file or directory: %s", filename)
downloadFromUrl(mp3Url)
}
}
}
}
func main() {
// Open an input file, exit on error.
inputFile, err := os.Open("bbc.txt")
if err != nil {
log.Fatal("Error opening input file:", err)
}
var regularExpression = regexp.MustCompile("http://downloads.bbc.co.uk/podcasts/[^.]*.mp3")
// Closes the file when we leave the scope of the current function,
// this makes sure we never forget to close the file if the
// function can exit in multiple places.
defer inputFile.Close()
scanner := bufio.NewScanner(inputFile)
// scanner.Scan() advances to the next token returning false if an error was encountered
for scanner.Scan() {
var websiteUrl = scanner.Text()
fmt.Println(websiteUrl)
downloadUrl(regularExpression, websiteUrl)
}
// When finished scanning if any error other than io.EOF occured
// it will be returned by scanner.Err().
if err := scanner.Err(); err != nil {
log.Fatal(scanner.Err())
}
}作者: DDD888 时间: 11-12-2014 14:28
Here is the source code to concurrent download url and mp3 files in golang.
