1 /** 2 * Copyright: Copyright (c) 2010-2011 Jacob Carlborg. All rights reserved. 3 * Authors: Jacob Carlborg 4 * Version: Initial created: Nov 8, 2010 5 * License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost Software License 1.0) 6 */ 7 module dvm.commands.Fetch; 8 9 import std.algorithm : splitter; 10 import std..string : split; 11 import std.regex; 12 13 import tango.core.Exception; 14 import tango.io.device.File; 15 import tango.io.model.IConduit; 16 import Path = tango.io.Path; 17 import tango.net.InternetAddress; 18 import tango.net.device.Socket; 19 import tango.net.http.HttpGet; 20 import tango.net.http.HttpConst; 21 22 import dvm.commands.Command; 23 import mambo.core._; 24 import dvm.util._; 25 import dvm.dvm._; 26 27 class Fetch : Command 28 { 29 this (string name, string summary = "") 30 { 31 super(name, summary); 32 } 33 34 this () 35 { 36 super("fetch", "Fetch a D compiler but don't install it."); 37 } 38 39 override void execute () 40 { 41 if (args.any && args.first == "dmc") 42 fetchDMC; 43 44 else 45 { 46 auto version_ = getDMDVersion(); 47 string filename = buildFilename(version_); 48 string url = buildUrl(version_, filename); 49 fetch(url, Path.join(".", filename).assumeUnique); 50 } 51 } 52 53 protected: 54 55 enum userAgent = buildUserAgent(); 56 enum dmcArchiveName = "dm852c.zip"; 57 58 void fetchDMC (string destinationPath=".") 59 { 60 auto url = "http://ftp.digitalmars.com/Digital_Mars_C++/Patch/" ~ dmcArchiveName; 61 fetch(url, Path.join(destinationPath, dmcArchiveName).assumeUnique); 62 } 63 64 void fetch (string source, string destination) 65 { 66 if (Path.exists(destination)) 67 return; 68 69 if (options.verbose) 70 { 71 println("Fetching:"); 72 println(options.indentation, "source: ", source); 73 println(options.indentation, "destination: ", destination, '\n'); 74 } 75 76 else 77 println("Fetching: ", source); 78 79 createPath(Path.parse(destination).folder); 80 writeFile(downloadFile(source), destination); 81 } 82 83 void[] downloadFile (string url) 84 { 85 static void print (A...)(A args) 86 { 87 import tango.io.Stdout; 88 89 static enum string fmt = "{}{}{}{}{}{}{}{}" 90 "{}{}{}{}{}{}{}{}" 91 "{}{}{}{}{}{}{}{}"; 92 93 static assert (A.length <= fmt.length / 2, "mambo.io.print :: too many arguments"); 94 95 Stdout.format(fmt[0 .. args.length * 2], args).flush; 96 } 97 98 auto page = new HttpGet(url); 99 page.setTimeout(30f); 100 page.getRequestHeaders.add(HttpHeader.UserAgent, userAgent); 101 auto buffer = page.open; 102 103 scope(exit) 104 page.close; 105 106 checkPageStatus(page, url); 107 108 // load in chunks in order to display progress 109 int contentLength = page.getResponseHeaders.getInt(HttpHeader.ContentLength); 110 111 enum width = 40; 112 int num = width; 113 114 version (Posix) 115 { 116 enum clearLine = "\033[1K"; // clear backwards 117 enum saveCursor = "\0337"; 118 enum restoreCursor = "\0338"; 119 } 120 121 else 122 { 123 enum clearLine = "\r"; 124 125 // Leaving these empty string causes a linker error: 126 // http://d.puremagic.com/issues/show_bug.cgi?id=4315 127 enum saveCursor = "\0"; 128 enum restoreCursor = "\0"; 129 } 130 131 print(saveCursor); 132 133 int bytesLeft = contentLength; 134 int chunkSize = bytesLeft / num; 135 136 while (bytesLeft > 0) 137 { 138 buffer.load(chunkSize > bytesLeft ? bytesLeft : chunkSize); 139 bytesLeft -= chunkSize; 140 int i = 0; 141 142 print(clearLine ~ restoreCursor ~ saveCursor); 143 print("["); 144 145 for ( ; i < (width - num); i++) 146 print("="); 147 148 print('>'); 149 150 for ( ; i < width; i++) 151 print(" "); 152 153 print("]"); 154 print(" ", (contentLength - bytesLeft) / 1024, "/", contentLength / 1024, " KB"); 155 156 num--; 157 } 158 159 println(restoreCursor); 160 println(); 161 162 return buffer.slice; 163 } 164 165 void writeFile (void[] data, string filename) 166 { 167 auto file = new File(filename, File.WriteCreate); 168 scope(exit) file.close(); 169 file.write(data); 170 } 171 172 string buildFilename (string ver="") 173 { 174 if (ver == "") 175 ver = getDMDVersion; 176 177 auto filename = format("dmd.{}.{}.zip", ver, platformForArchive); 178 auto url = buildUrl(ver, filename); 179 180 return urlExists(url) ? filename : format("dmd.{}.zip", ver); 181 } 182 183 string buildUrl (string version_, string filename) 184 { 185 auto url = dlangUrl(version_, filename); 186 187 if (urlExists(url)) 188 return url; 189 190 return digitalMarsUrl(filename); 191 } 192 193 string digitalMarsUrl (string filename) 194 { 195 return "http://ftp.digitalmars.com/" ~ filename; 196 } 197 198 string dlangUrl (string version_, string filename) 199 { 200 enum baseUrl = "http://downloads.dlang.org/"; 201 string releases = "releases"; 202 203 if (isPreRelease(version_)) 204 { 205 releases = "pre-releases"; 206 version_ = version_.split("-").first; 207 } 208 209 return format(baseUrl ~ "{}/{}.x/{}/{}", releases, version_.first, version_, filename); 210 } 211 212 void createPath (string path) 213 { 214 if (!Path.exists(path)) 215 Path.createPath(path); 216 } 217 218 void checkPageStatus (HttpGet page, string url) 219 { 220 if (page.getStatus == 404) 221 throw new IOException(format(`The resource with URL "{}" could not be found.`, url)); 222 223 else if (!page.isResponseOK) 224 throw new IOException(format(`An unexpected error occurred. The resource "{}" responded with the message "{}" and the status code {}.`, url, page.getResponse.getReason, page.getResponse.getStatus)); 225 } 226 227 bool urlExists (string url, float timeout = 5f) 228 { 229 scope page = new HttpGet(url); 230 page.setTimeout(timeout); 231 page.enableRedirect(); 232 page.open(); 233 234 return page.isResponseOK; 235 } 236 237 string getDMDVersion () 238 { 239 if (options.latest) 240 { 241 auto vers = getDVersion; 242 return args.first = vers ~ "." ~ getLatestDMDVersion(vers); 243 } 244 245 else 246 { 247 validateArguments(); 248 return args.first; 249 } 250 } 251 252 string getDVersion () 253 { 254 return args.empty() ? "2" : args.first; 255 } 256 257 string getLatestDMDVersion (string dVersion) 258 { 259 auto dmdPattern = r"(?:dmd\." ~ dVersion ~ r"\.([\d.]+)\.zip)"; 260 auto pattern = regex(r"http:\/\/downloads\.dlang\.org\/releases\/(?:\d+)\/" ~ dmdPattern); 261 262 if (auto result = getLatestDMDVersionImpl(pattern)) 263 return result; 264 265 pattern = regex(r"http:\/\/ftp\.digitalmars\.com\/" ~ dmdPattern); 266 267 if (auto result = getLatestDMDVersionImpl(pattern)) 268 return result; 269 270 throw new DvmException("Failed to get the latest DMD version.", __FILE__, __LINE__); 271 } 272 273 private string getLatestDMDVersionImpl (Regex!(char) regex) 274 { 275 scope page = new HttpGet("http://dlang.org/download.html"); 276 auto content = cast(string) page.read; 277 278 string vers = null; 279 280 foreach (line ; content.splitter('\n')) 281 { 282 auto match = line.matchFirst(regex); 283 284 if (match.any && match[1] > vers) 285 vers = match[1]; 286 } 287 288 return vers; 289 } 290 291 void validateArguments (string errorMessage = null) 292 { 293 if (errorMessage.isEmpty) 294 errorMessage = "Cannot fetch a compiler without specifying a version"; 295 296 if (args.empty) 297 throw new DvmException(errorMessage, __FILE__, __LINE__); 298 } 299 300 bool isPreRelease (string version_) 301 { 302 immutable parts = version_.split("-"); 303 304 if (parts.length != 2) 305 return false; 306 307 immutable suffix = parts[1]; 308 309 return suffix.first == 'b' || suffix.startsWith("rc"); 310 } 311 312 string platformForArchive() 313 { 314 auto platform = options.platform; 315 316 version (FreeBSD) 317 platform ~= options.is64bit ? "-64" : "-32"; 318 319 return platform; 320 } 321 } 322 323 string buildUserAgent () 324 { 325 import std.system; 326 import std..string; 327 328 version (X86) auto architecture = "i386"; 329 else version (X86_64) auto architecture = "x86_64"; 330 else static assert("Unhandled platform"); 331 332 return format("dvm/%s (%s-%s)", dvm.dvm.Version.Version, architecture, os); 333 } 334 335 template FetchImpl () 336 { 337 void execute () 338 { 339 auto filename = buildFilename; 340 auto url = buildUrl(filename); 341 fetch(url, join(".", filename)); 342 } 343 344 protected void fetch (string source, string destination) 345 { 346 writeFile(downloadFile(source), destination); 347 } 348 349 private void[] downloadFile (string url) 350 { 351 auto page = new HttpGet(url); 352 353 if (!page.isResponseOK()) 354 throw new IOException(format("{}", page.getResponse.getStatus)); 355 356 return page.read; 357 } 358 359 private void writeFile (void[] data, string filename) 360 { 361 auto file = new File(filename, File.WriteCreate); 362 file.write(data); 363 } 364 365 private string buildFilename () 366 { 367 return "dmd." ~ args.first ~ ".zip"; 368 } 369 370 private string buildUrl (string filename) 371 { 372 return "http://ftp.digitalmars.com/" ~ filename; 373 } 374 }