1 /**
2  * Copyright: Copyright (c) 2010-2011 Jacob Carlborg. All rights reserved.
3  * Authors: Jacob Carlborg
4  * Version: Initial created: Nov 8, 2010
5  * License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost Software License 1.0)
6  */
7 module dvm.commands.Fetch;
8 
9 import std.algorithm : splitter;
10 import std..string : split;
11 import std.regex;
12 
13 import tango.core.Exception;
14 import tango.io.device.File;
15 import tango.io.model.IConduit;
16 import Path = tango.io.Path;
17 import tango.net.InternetAddress;
18 import tango.net.device.Socket;
19 import tango.net.http.HttpGet;
20 import tango.net.http.HttpConst;
21 
22 import dvm.commands.Command;
23 import mambo.core._;
24 import dvm.util._;
25 import dvm.dvm._;
26 
27 class Fetch : Command
28 {
29     this (string name, string summary = "")
30     {
31         super(name, summary);
32     }
33 
34     this ()
35     {
36         super("fetch", "Fetch a D compiler but don't install it.");
37     }
38 
39     override void execute ()
40     {
41         if (args.any && args.first == "dmc")
42             fetchDMC;
43 
44         else
45         {
46             auto version_ = getDMDVersion();
47             string filename = buildFilename(version_);
48             string url = buildUrl(version_, filename);
49             fetch(url, Path.join(".", filename).assumeUnique);
50         }
51     }
52 
53 protected:
54 
55     enum userAgent = buildUserAgent();
56     enum dmcArchiveName = "dm852c.zip";
57 
58     void fetchDMC (string destinationPath=".")
59     {
60         auto url = "http://ftp.digitalmars.com/Digital_Mars_C++/Patch/" ~ dmcArchiveName;
61         fetch(url, Path.join(destinationPath, dmcArchiveName).assumeUnique);
62     }
63 
64     void fetch (string source, string destination)
65     {
66         if (Path.exists(destination))
67             return;
68 
69         if (options.verbose)
70         {
71             println("Fetching:");
72             println(options.indentation, "source: ", source);
73             println(options.indentation, "destination: ", destination, '\n');
74         }
75 
76         else
77             println("Fetching: ", source);
78 
79         createPath(Path.parse(destination).folder);
80         writeFile(downloadFile(source), destination);
81     }
82 
83     void[] downloadFile (string url)
84     {
85         static void print (A...)(A args)
86         {
87             import tango.io.Stdout;
88 
89             static enum string fmt = "{}{}{}{}{}{}{}{}"
90                                      "{}{}{}{}{}{}{}{}"
91                                      "{}{}{}{}{}{}{}{}";
92 
93             static assert (A.length <= fmt.length / 2, "mambo.io.print :: too many arguments");
94 
95             Stdout.format(fmt[0 .. args.length * 2], args).flush;
96         }
97 
98         auto page = new HttpGet(url);
99         page.setTimeout(30f);
100         page.getRequestHeaders.add(HttpHeader.UserAgent, userAgent);
101         auto buffer = page.open;
102 
103         scope(exit)
104             page.close;
105 
106         checkPageStatus(page, url);
107 
108         // load in chunks in order to display progress
109         int contentLength = page.getResponseHeaders.getInt(HttpHeader.ContentLength);
110 
111         enum width = 40;
112         int num = width;
113 
114         version (Posix)
115         {
116             enum clearLine = "\033[1K"; // clear backwards
117             enum saveCursor = "\0337";
118             enum restoreCursor = "\0338";
119         }
120 
121         else
122         {
123             enum clearLine = "\r";
124 
125             // Leaving these empty string causes a linker error:
126             // http://d.puremagic.com/issues/show_bug.cgi?id=4315
127             enum saveCursor = "\0";
128             enum restoreCursor = "\0";
129         }
130 
131         print(saveCursor);
132 
133         int bytesLeft = contentLength;
134         int chunkSize = bytesLeft / num;
135 
136         while (bytesLeft > 0)
137         {
138             buffer.load(chunkSize > bytesLeft ? bytesLeft : chunkSize);
139             bytesLeft -= chunkSize;
140             int i = 0;
141 
142             print(clearLine ~ restoreCursor ~ saveCursor);
143             print("[");
144 
145             for ( ; i < (width - num); i++)
146                 print("=");
147 
148             print('>');
149 
150             for ( ; i < width; i++)
151                 print(" ");
152 
153             print("]");
154             print(" ", (contentLength - bytesLeft) / 1024, "/", contentLength / 1024, " KB");
155 
156             num--;
157         }
158 
159         println(restoreCursor);
160         println();
161 
162         return buffer.slice;
163     }
164 
165     void writeFile (void[] data, string filename)
166     {
167         auto file = new File(filename, File.WriteCreate);
168         scope(exit) file.close();
169         file.write(data);
170     }
171 
172     string buildFilename (string ver="")
173     {
174         if (ver == "")
175             ver = getDMDVersion;
176 
177         auto filename = format("dmd.{}.{}.zip", ver, platformForArchive);
178         auto url = buildUrl(ver, filename);
179 
180         return urlExists(url) ? filename : format("dmd.{}.zip", ver);
181     }
182 
183     string buildUrl (string version_, string filename)
184     {
185         auto url = dlangUrl(version_, filename);
186 
187         if (urlExists(url))
188             return url;
189 
190         return digitalMarsUrl(filename);
191     }
192 
193     string digitalMarsUrl (string filename)
194     {
195         return "http://ftp.digitalmars.com/" ~ filename;
196     }
197 
198     string dlangUrl (string version_, string filename)
199     {
200         enum baseUrl = "http://downloads.dlang.org/";
201         string releases = "releases";
202 
203         if (isPreRelease(version_))
204         {
205             releases = "pre-releases";
206             version_ = version_.split("-").first;
207         }
208 
209         return format(baseUrl ~ "{}/{}.x/{}/{}", releases, version_.first, version_, filename);
210     }
211 
212     void createPath (string path)
213     {
214         if (!Path.exists(path))
215             Path.createPath(path);
216     }
217 
218     void checkPageStatus (HttpGet page, string url)
219     {
220         if (page.getStatus == 404)
221             throw new IOException(format(`The resource with URL "{}" could not be found.`, url));
222 
223         else if (!page.isResponseOK)
224             throw new IOException(format(`An unexpected error occurred. The resource "{}" responded with the message "{}" and the status code {}.`, url, page.getResponse.getReason, page.getResponse.getStatus));
225     }
226 
227     bool urlExists (string url, float timeout = 5f)
228     {
229         scope page = new HttpGet(url);
230         page.setTimeout(timeout);
231         page.enableRedirect();
232         page.open();
233 
234         return page.isResponseOK;
235     }
236 
237     string getDMDVersion ()
238     {
239         if (options.latest)
240         {
241             auto vers = getDVersion;
242             return args.first = vers ~ "." ~ getLatestDMDVersion(vers);
243         }
244 
245         else
246         {
247             validateArguments();
248             return args.first;
249         }
250     }
251 
252     string getDVersion ()
253     {
254         return args.empty() ? "2" : args.first;
255     }
256 
257     string getLatestDMDVersion (string dVersion)
258     {
259         auto dmdPattern = r"(?:dmd\." ~ dVersion ~ r"\.([\d.]+)\.zip)";
260         auto pattern = regex(r"http:\/\/downloads\.dlang\.org\/releases\/(?:\d+)\/" ~ dmdPattern);
261 
262         if (auto result = getLatestDMDVersionImpl(pattern))
263             return result;
264 
265         pattern = regex(r"http:\/\/ftp\.digitalmars\.com\/" ~ dmdPattern);
266 
267         if (auto result = getLatestDMDVersionImpl(pattern))
268             return result;
269 
270         throw new DvmException("Failed to get the latest DMD version.", __FILE__, __LINE__);
271     }
272 
273     private string getLatestDMDVersionImpl (Regex!(char) regex)
274     {
275         scope page = new HttpGet("http://dlang.org/download.html");
276         auto content = cast(string) page.read;
277 
278         string vers = null;
279 
280         foreach (line ; content.splitter('\n'))
281         {
282             auto match = line.matchFirst(regex);
283 
284             if (match.any && match[1] > vers)
285                 vers = match[1];
286         }
287 
288         return vers;
289     }
290 
291     void validateArguments (string errorMessage = null)
292     {
293         if (errorMessage.isEmpty)
294             errorMessage = "Cannot fetch a compiler without specifying a version";
295 
296         if (args.empty)
297             throw new DvmException(errorMessage, __FILE__, __LINE__);
298     }
299 
300     bool isPreRelease (string version_)
301     {
302         immutable parts = version_.split("-");
303 
304         if (parts.length != 2)
305             return false;
306 
307         immutable suffix = parts[1];
308 
309         return suffix.first == 'b' || suffix.startsWith("rc");
310     }
311 
312     string platformForArchive()
313     {
314         auto platform = options.platform;
315 
316         version (FreeBSD)
317             platform ~= options.is64bit ? "-64" : "-32";
318 
319         return platform;
320     }
321 }
322 
323 string buildUserAgent ()
324 {
325     import std.system;
326     import std..string;
327 
328     version (X86) auto architecture = "i386";
329     else version (X86_64) auto architecture = "x86_64";
330     else static assert("Unhandled platform");
331 
332     return format("dvm/%s (%s-%s)", dvm.dvm.Version.Version, architecture, os);
333 }
334 
335 template FetchImpl ()
336 {
337     void execute ()
338     {
339         auto filename = buildFilename;
340         auto url = buildUrl(filename);
341         fetch(url, join(".", filename));
342     }
343 
344     protected void fetch (string source, string destination)
345     {
346         writeFile(downloadFile(source), destination);
347     }
348 
349     private void[] downloadFile (string url)
350     {
351         auto page = new HttpGet(url);
352 
353         if (!page.isResponseOK())
354             throw new IOException(format("{}", page.getResponse.getStatus));
355 
356         return page.read;
357     }
358 
359     private void writeFile (void[] data, string filename)
360     {
361         auto file = new File(filename, File.WriteCreate);
362         file.write(data);
363     }
364 
365     private string buildFilename ()
366     {
367         return "dmd." ~ args.first ~ ".zip";
368     }
369 
370     private string buildUrl (string filename)
371     {
372         return "http://ftp.digitalmars.com/" ~ filename;
373     }
374 }