-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathdownit.nim
191 lines (155 loc) · 8.85 KB
/
downit.nim
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
## The `Downloader` object holds all the downloads data, `initDownloader` takes the root directory for all the downloads (or `""` if none) and the poll timeout (by default 1 ms).
## After initializing it, downloading something it's just as easy as calling `download`, procedure that takes the url, path and an optional name, if not given the path will be the name. Setting a name is useful to identify a download and makes changing the path a lot easier.
## You can also make a GET request using the `request` procedure, passing the url and optionally a name.
## After making a download/request you can use the `downloading`, `downloaded`, `finished` and `failed` procedures to check wheter a download/request finished, is still in progress or failed.
## ```nim
## # Documentation downloader
## var downloader = initDownloader("./docs")
## downloader.download("https://nim-lang.org/docs/os.html", "os.html", "os")
## downloader.request("https://nim-lang.org/docs/strformat.html", "strformat")
##
## while true:
## downloader.update() # Poll events and check if downloads are complete
##
## if downloader.succeed("os") and downloader.succeed("strformat"):
## echo readFile(downloader.getPath("os").get())
## echo downloader.getBody("strformat").get()
## break
##
## elif downloader.getError("os").isSome: # You can also do downloader.getState("os").get() == DownloadError
## raise downloader.getError("os").get()
## ```
import std/[asyncdispatch, httpclient, options, tables, os]
export httpclient, options
type
DownloadState* = enum
Downloading, Downloaded, DownloadError
Download* = object
url*, path*: string
state*: DownloadState
error*: ref Exception
downFuture*: Future[void]
requestFuture*: Future[AsyncResponse]
Downloader* = object
dir*: string ## Root directory for all downloads
timeout*: int ## Poll events timeout
proxy: Proxy
downTable*: Table[string, Download]
proc initDownload(url, path: string, state: DownloadState, error: ref Exception = nil, requestFuture: Future[AsyncResponse] = nil, downFuture: Future[void] = nil): Download =
Download(url: url, path: path, state: state, error: error, requestFuture: requestFuture, downFuture: downFuture)
proc setProxy*(self: var Downloader, proxy: string, proxyUser, proxyPassword = "") =
self.proxy = newProxy(proxy, auth = proxyUser & ':' & proxyPassword)
proc removeProxy*(self: var Downloader) =
self.proxy = nil
proc initDownloader*(dir: string, timeout = 1, proxy, proxyUser, proxyPassword = ""): Downloader =
## Initializes a Downloader object and creates `dir`.
dir.createDir()
result = Downloader(dir: dir, timeout: timeout)
if proxy.len > 0:
result.setProxy(proxy, proxyUser, proxyPassword)
proc exists*(self: Downloader, name: string): bool =
## Returns true if a download with `name` exists.
name in self.downTable
proc getError*(self: Downloader, name: string): Option[ref Exception] =
## Returns the exception of `name` if it had a `DownloadError`.
if self.exists(name) and self.downTable[name].state == DownloadError:
result = self.downTable[name].error.some()
proc getErrorMsg*(self: Downloader, name: string): Option[string] =
## Returns the error message of `name` if it had a `DownloadError`.
if (let error = self.getError(name); error.isSome):
result = error.get().msg.some()
proc isDownload*(self: Downloader, name: string): bool =
self.exists(name) and self.downTable[name].path.len > 0
proc isRequest*(self: Downloader, name: string): bool =
self.exists(name) and self.downTable[name].path.len == 0
proc getPath*(self: Downloader, name: string, joinDir = true): Option[string] =
## Returns the path of `name` if it exists, joined with the downloader's root dir if `joinDir` is true otherwise returns the raw path.
## Returns none for requests.
if self.exists(name) and self.isDownload(name):
if joinDir:
result = some(self.dir / self.downTable[name].path)
else:
result = self.downTable[name].path.some()
proc getURL*(self: Downloader, name: string): Option[string] =
## Returns the url of `name` if it exists.
if self.exists(name):
result = self.downTable[name].url.some()
proc getState*(self: Downloader, name: string): Option[DownloadState] =
## Returns the state of `name` if it exists.
if self.exists(name):
result = self.downTable[name].state.some()
proc getResponse*(self: Downloader, name: string): Option[AsyncResponse] =
## Returns the AsyncRespones of `name` if it finished.
## Returns none for downloads.
if self.exists(name) and self.isRequest(name) and self.downTable[name].state == Downloaded:
result = self.downTable[name].requestFuture.read().some()
proc getBody*(self: Downloader, name: string): Option[string] =
## Returns the body of `name` if it finished.
## Returns none for downloads.
if (let response = self.getResponse(name); response.isSome and response.get().body.finished):
result = response.get().body.read().some() # body is procedure that returns a Future[string]
proc remove*(self: var Downloader, name: string) =
## Removes `name`'s file if it exists.
if self.isDownload(name) and self.getPath(name).get().fileExists():
self.getPath(name).get().removeFile()
proc succeed*(self: Downloader, name: string): bool =
## Returns true if `name` was downloaded/requested successfully, the path must exist if it is a download.
self.exists(name) and self.getState(name).get() == Downloaded and (self.isRequest(name) or self.getPath(name).get().fileExists())
proc finished*(self: Downloader, name: string): bool =
## Returns true if `name` succeed or failed.
self.exists(name) and self.getState(name).get() in {Downloaded, DownloadError}
proc failed*(self: Downloader, name: string): bool =
## Returns true if `name` had a DownloadError.
self.exists(name) and self.getState(name).get() == DownloadError
proc running*(self: Downloader, name: string): bool =
## Returns true if `name` is being downloaded/requested.
self.exists(name) and self.getState(name).get() == Downloading
proc downTable*(self: Downloader): Table[string, Download] =
self.downTable
proc downloadImpl*(url, path: string, proxy: Proxy, onProgressChanged: ProgressChangedProc[Future[void]] = nil): Future[void] {.async.} =
let client = newAsyncHttpClient(proxy = proxy)
client.onProgressChanged = onProgressChanged
await client.downloadFile(url, path)
client.close()
proc download*(self: var Downloader, url, path: string, name = "", replace = false, onProgressChanged: ProgressChangedProc[Future[void]] = nil) =
## Starts an asynchronous download of `url` to `path`.
## `path` will be used as the name if `name` is empty.
## If `replace` is set to true and the file is downloaded overwrite it, otherwise if the file is downloaded and `replace` is false do nothing.
if not replace and self.succeed(name): return
path.splitPath.head.createDir()
self.downTable[if name.len > 0: name else: path] = initDownload(url, path, Downloading, downFuture = downloadImpl(url, self.dir / path, self.proxy, onProgressChanged))
proc requestImpl*(url: string, proxy: Proxy): Future[AsyncResponse] {.async.} =
let client = newAsyncHttpClient(proxy = proxy)
result = await client.get(url)
yield result.body
client.close()
proc request*(self: var Downloader, url: string, name = "") =
## Starts an asynchronous GET request of `url`.
## `url` will be used as the name if `name` is empty.
self.downTable[if name.len > 0: name else: url] = initDownload(url, "", Downloading, requestFuture = requestImpl(url, self.proxy))
proc downloadAgain*(self: var Downloader, name: string) =
## Downloads `name` again if it exists, does nothing otherwise.
if self.exists(name) and self.isDownload(name):
self.download(self.getURL(name).get(), self.getPath(name, joinDir = false).get(), name, replace = true)
proc requestAgain*(self: var Downloader, name: string) =
## Requests `name` again if it exists, does nothing otherwise.
if self.exists(name) and self.isRequest(name):
self.request(self.getURL(name).get(), name)
proc update*(self: var Downloader) =
## Poll for any outstanding events and check if any download/request is complete.
waitFor sleepAsync(self.timeout)
for name, data in self.downTable:
if data.state == Downloading:
if not data.downFuture.isNil and data.downFuture.finished:
if data.downFuture.failed:
self.remove(name)
self.downTable[name].state = DownloadError
self.downTable[name].error = data.downFuture.readError()
elif self.getPath(name).get().fileExists():
self.downTable[name].state = Downloaded
elif not data.requestFuture.isNil and data.requestFuture.finished:
if data.requestFuture.failed:
self.downTable[name].state = DownloadError
self.downTable[name].error = data.requestFuture.readError()
else:
self.downTable[name].state = Downloaded