import http = require('http') import https = require('https') import ifm = require('./interfaces') import pm = require('./proxy') let tunnel: any export enum HttpCodes { OK = 200, MultipleChoices = 300, MovedPermanently = 301, ResourceMoved = 302, SeeOther = 303, NotModified = 304, UseProxy = 305, SwitchProxy = 306, TemporaryRedirect = 307, PermanentRedirect = 308, BadRequest = 400, Unauthorized = 401, PaymentRequired = 402, Forbidden = 403, NotFound = 404, MethodNotAllowed = 405, NotAcceptable = 406, ProxyAuthenticationRequired = 407, RequestTimeout = 408, Conflict = 409, Gone = 410, TooManyRequests = 429, InternalServerError = 500, NotImplemented = 501, BadGateway = 502, ServiceUnavailable = 503, GatewayTimeout = 504 } export enum Headers { Accept = 'accept', ContentType = 'content-type' } export enum MediaTypes { ApplicationJson = 'application/json' } /** * Returns the proxy URL, depending upon the supplied url and proxy environment variables. * @param serverUrl The server URL where the request will be sent. For example, https://api.github.com */ export function getProxyUrl(serverUrl: string): string { let proxyUrl = pm.getProxyUrl(new URL(serverUrl)) return proxyUrl ? proxyUrl.href : '' } const HttpRedirectCodes: number[] = [ HttpCodes.MovedPermanently, HttpCodes.ResourceMoved, HttpCodes.SeeOther, HttpCodes.TemporaryRedirect, HttpCodes.PermanentRedirect ] const HttpResponseRetryCodes: number[] = [ HttpCodes.BadGateway, HttpCodes.ServiceUnavailable, HttpCodes.GatewayTimeout ] const RetryableHttpVerbs: string[] = ['OPTIONS', 'GET', 'DELETE', 'HEAD'] const ExponentialBackoffCeiling = 10 const ExponentialBackoffTimeSlice = 5 export class HttpClientResponse implements ifm.IHttpClientResponse { constructor(message: http.IncomingMessage) { this.message = message } public message: http.IncomingMessage readBody(): Promise { return new Promise(async (resolve, reject) => { let output = Buffer.alloc(0) this.message.on('data', (chunk: Buffer) => { output = Buffer.concat([output, chunk]) }) this.message.on('end', () => { resolve(output.toString()) }) }) } } export function isHttps(requestUrl: string) { let parsedUrl: URL = new URL(requestUrl) return parsedUrl.protocol === 'https:' } export class HttpClient { userAgent: string | undefined handlers: ifm.IRequestHandler[] requestOptions: ifm.IRequestOptions private _ignoreSslError: boolean = false private _socketTimeout: number private _allowRedirects: boolean = true private _allowRedirectDowngrade: boolean = false private _maxRedirects: number = 50 private _allowRetries: boolean = false private _maxRetries: number = 1 private _agent private _proxyAgent private _keepAlive: boolean = false private _disposed: boolean = false constructor( userAgent?: string, handlers?: ifm.IRequestHandler[], requestOptions?: ifm.IRequestOptions ) { this.userAgent = userAgent this.handlers = handlers || [] this.requestOptions = requestOptions if (requestOptions) { if (requestOptions.ignoreSslError != null) { this._ignoreSslError = requestOptions.ignoreSslError } this._socketTimeout = requestOptions.socketTimeout if (requestOptions.allowRedirects != null) { this._allowRedirects = requestOptions.allowRedirects } if (requestOptions.allowRedirectDowngrade != null) { this._allowRedirectDowngrade = requestOptions.allowRedirectDowngrade } if (requestOptions.maxRedirects != null) { this._maxRedirects = Math.max(requestOptions.maxRedirects, 0) } if (requestOptions.keepAlive != null) { this._keepAlive = requestOptions.keepAlive } if (requestOptions.allowRetries != null) { this._allowRetries = requestOptions.allowRetries } if (requestOptions.maxRetries != null) { this._maxRetries = requestOptions.maxRetries } } } public options( requestUrl: string, additionalHeaders?: ifm.IHeaders ): Promise { return this.request('OPTIONS', requestUrl, null, additionalHeaders || {}) } public get( requestUrl: string, additionalHeaders?: ifm.IHeaders ): Promise { return this.request('GET', requestUrl, null, additionalHeaders || {}) } public del( requestUrl: string, additionalHeaders?: ifm.IHeaders ): Promise { return this.request('DELETE', requestUrl, null, additionalHeaders || {}) } public post( requestUrl: string, data: string, additionalHeaders?: ifm.IHeaders ): Promise { return this.request('POST', requestUrl, data, additionalHeaders || {}) } public patch( requestUrl: string, data: string, additionalHeaders?: ifm.IHeaders ): Promise { return this.request('PATCH', requestUrl, data, additionalHeaders || {}) } public put( requestUrl: string, data: string, additionalHeaders?: ifm.IHeaders ): Promise { return this.request('PUT', requestUrl, data, additionalHeaders || {}) } public head( requestUrl: string, additionalHeaders?: ifm.IHeaders ): Promise { return this.request('HEAD', requestUrl, null, additionalHeaders || {}) } public sendStream( verb: string, requestUrl: string, stream: NodeJS.ReadableStream, additionalHeaders?: ifm.IHeaders ): Promise { return this.request(verb, requestUrl, stream, additionalHeaders) } /** * Gets a typed object from an endpoint * Be aware that not found returns a null. Other errors (4xx, 5xx) reject the promise */ public async getJson( requestUrl: string, additionalHeaders: ifm.IHeaders = {} ): Promise> { additionalHeaders[Headers.Accept] = this._getExistingOrDefaultHeader( additionalHeaders, Headers.Accept, MediaTypes.ApplicationJson ) let res: ifm.IHttpClientResponse = await this.get( requestUrl, additionalHeaders ) return this._processResponse(res, this.requestOptions) } public async postJson( requestUrl: string, obj: any, additionalHeaders: ifm.IHeaders = {} ): Promise> { let data: string = JSON.stringify(obj, null, 2) additionalHeaders[Headers.Accept] = this._getExistingOrDefaultHeader( additionalHeaders, Headers.Accept, MediaTypes.ApplicationJson ) additionalHeaders[Headers.ContentType] = this._getExistingOrDefaultHeader( additionalHeaders, Headers.ContentType, MediaTypes.ApplicationJson ) let res: ifm.IHttpClientResponse = await this.post( requestUrl, data, additionalHeaders ) return this._processResponse(res, this.requestOptions) } public async putJson( requestUrl: string, obj: any, additionalHeaders: ifm.IHeaders = {} ): Promise> { let data: string = JSON.stringify(obj, null, 2) additionalHeaders[Headers.Accept] = this._getExistingOrDefaultHeader( additionalHeaders, Headers.Accept, MediaTypes.ApplicationJson ) additionalHeaders[Headers.ContentType] = this._getExistingOrDefaultHeader( additionalHeaders, Headers.ContentType, MediaTypes.ApplicationJson ) let res: ifm.IHttpClientResponse = await this.put( requestUrl, data, additionalHeaders ) return this._processResponse(res, this.requestOptions) } public async patchJson( requestUrl: string, obj: any, additionalHeaders: ifm.IHeaders = {} ): Promise> { let data: string = JSON.stringify(obj, null, 2) additionalHeaders[Headers.Accept] = this._getExistingOrDefaultHeader( additionalHeaders, Headers.Accept, MediaTypes.ApplicationJson ) additionalHeaders[Headers.ContentType] = this._getExistingOrDefaultHeader( additionalHeaders, Headers.ContentType, MediaTypes.ApplicationJson ) let res: ifm.IHttpClientResponse = await this.patch( requestUrl, data, additionalHeaders ) return this._processResponse(res, this.requestOptions) } /** * Makes a raw http request. * All other methods such as get, post, patch, and request ultimately call this. * Prefer get, del, post and patch */ public async request( verb: string, requestUrl: string, data: string | NodeJS.ReadableStream, headers: ifm.IHeaders ): Promise { if (this._disposed) { throw new Error('Client has already been disposed.') } let parsedUrl = new URL(requestUrl) let info: ifm.IRequestInfo = this._prepareRequest(verb, parsedUrl, headers) // Only perform retries on reads since writes may not be idempotent. let maxTries: number = this._allowRetries && RetryableHttpVerbs.indexOf(verb) != -1 ? this._maxRetries + 1 : 1 let numTries: number = 0 let response: HttpClientResponse while (numTries < maxTries) { response = await this.requestRaw(info, data) // Check if it's an authentication challenge if ( response && response.message && response.message.statusCode === HttpCodes.Unauthorized ) { let authenticationHandler: ifm.IRequestHandler for (let i = 0; i < this.handlers.length; i++) { if (this.handlers[i].canHandleAuthentication(response)) { authenticationHandler = this.handlers[i] break } } if (authenticationHandler) { return authenticationHandler.handleAuthentication(this, info, data) } else { // We have received an unauthorized response but have no handlers to handle it. // Let the response return to the caller. return response } } let redirectsRemaining: number = this._maxRedirects while ( HttpRedirectCodes.indexOf(response.message.statusCode) != -1 && this._allowRedirects && redirectsRemaining > 0 ) { const redirectUrl: string | null = response.message.headers['location'] if (!redirectUrl) { // if there's no location to redirect to, we won't break } let parsedRedirectUrl = new URL(redirectUrl) if ( parsedUrl.protocol == 'https:' && parsedUrl.protocol != parsedRedirectUrl.protocol && !this._allowRedirectDowngrade ) { throw new Error( 'Redirect from HTTPS to HTTP protocol. This downgrade is not allowed for security reasons. If you want to allow this behavior, set the allowRedirectDowngrade option to true.' ) } // we need to finish reading the response before reassigning response // which will leak the open socket. await response.readBody() // strip authorization header if redirected to a different hostname if (parsedRedirectUrl.hostname !== parsedUrl.hostname) { for (let header in headers) { // header names are case insensitive if (header.toLowerCase() === 'authorization') { delete headers[header] } } } // let's make the request with the new redirectUrl info = this._prepareRequest(verb, parsedRedirectUrl, headers) response = await this.requestRaw(info, data) redirectsRemaining-- } if (HttpResponseRetryCodes.indexOf(response.message.statusCode) == -1) { // If not a retry code, return immediately instead of retrying return response } numTries += 1 if (numTries < maxTries) { await response.readBody() await this._performExponentialBackoff(numTries) } } return response } /** * Needs to be called if keepAlive is set to true in request options. */ public dispose() { if (this._agent) { this._agent.destroy() } this._disposed = true } /** * Raw request. * @param info * @param data */ public requestRaw( info: ifm.IRequestInfo, data: string | NodeJS.ReadableStream ): Promise { return new Promise((resolve, reject) => { let callbackForResult = function ( err: any, res: ifm.IHttpClientResponse ) { if (err) { reject(err) } resolve(res) } this.requestRawWithCallback(info, data, callbackForResult) }) } /** * Raw request with callback. * @param info * @param data * @param onResult */ public requestRawWithCallback( info: ifm.IRequestInfo, data: string | NodeJS.ReadableStream, onResult: (err: any, res: ifm.IHttpClientResponse) => void ): void { let socket if (typeof data === 'string') { info.options.headers['Content-Length'] = Buffer.byteLength(data, 'utf8') } let callbackCalled: boolean = false let handleResult = (err: any, res: HttpClientResponse) => { if (!callbackCalled) { callbackCalled = true onResult(err, res) } } let req: http.ClientRequest = info.httpModule.request( info.options, (msg: http.IncomingMessage) => { let res: HttpClientResponse = new HttpClientResponse(msg) handleResult(null, res) } ) req.on('socket', sock => { socket = sock }) // If we ever get disconnected, we want the socket to timeout eventually req.setTimeout(this._socketTimeout || 3 * 60000, () => { if (socket) { socket.end() } handleResult(new Error('Request timeout: ' + info.options.path), null) }) req.on('error', function (err) { // err has statusCode property // res should have headers handleResult(err, null) }) if (data && typeof data === 'string') { req.write(data, 'utf8') } if (data && typeof data !== 'string') { data.on('close', function () { req.end() }) data.pipe(req) } else { req.end() } } /** * Gets an http agent. This function is useful when you need an http agent that handles * routing through a proxy server - depending upon the url and proxy environment variables. * @param serverUrl The server URL where the request will be sent. For example, https://api.github.com */ public getAgent(serverUrl: string): http.Agent { let parsedUrl = new URL(serverUrl) return this._getAgent(parsedUrl) } private _prepareRequest( method: string, requestUrl: URL, headers: ifm.IHeaders ): ifm.IRequestInfo { const info: ifm.IRequestInfo = {} info.parsedUrl = requestUrl const usingSsl: boolean = info.parsedUrl.protocol === 'https:' info.httpModule = usingSsl ? https : http const defaultPort: number = usingSsl ? 443 : 80 info.options = {} info.options.host = info.parsedUrl.hostname info.options.port = info.parsedUrl.port ? parseInt(info.parsedUrl.port) : defaultPort info.options.path = (info.parsedUrl.pathname || '') + (info.parsedUrl.search || '') info.options.method = method info.options.headers = this._mergeHeaders(headers) if (this.userAgent != null) { info.options.headers['user-agent'] = this.userAgent } info.options.agent = this._getAgent(info.parsedUrl) // gives handlers an opportunity to participate if (this.handlers) { this.handlers.forEach(handler => { handler.prepareRequest(info.options) }) } return info } private _mergeHeaders(headers: ifm.IHeaders): ifm.IHeaders { const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => ((c[k.toLowerCase()] = obj[k]), c), {}) if (this.requestOptions && this.requestOptions.headers) { return Object.assign( {}, lowercaseKeys(this.requestOptions.headers), lowercaseKeys(headers) ) } return lowercaseKeys(headers || {}) } private _getExistingOrDefaultHeader( additionalHeaders: ifm.IHeaders, header: string, _default: string ) { const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => ((c[k.toLowerCase()] = obj[k]), c), {}) let clientHeader: string if (this.requestOptions && this.requestOptions.headers) { clientHeader = lowercaseKeys(this.requestOptions.headers)[header] } return additionalHeaders[header] || clientHeader || _default } private _getAgent(parsedUrl: URL): http.Agent { let agent let proxyUrl: URL = pm.getProxyUrl(parsedUrl) let useProxy = proxyUrl && proxyUrl.hostname if (this._keepAlive && useProxy) { agent = this._proxyAgent } if (this._keepAlive && !useProxy) { agent = this._agent } // if agent is already assigned use that agent. if (!!agent) { return agent } const usingSsl = parsedUrl.protocol === 'https:' let maxSockets = 100 if (!!this.requestOptions) { maxSockets = this.requestOptions.maxSockets || http.globalAgent.maxSockets } if (useProxy) { // If using proxy, need tunnel if (!tunnel) { tunnel = require('tunnel') } const agentOptions = { maxSockets: maxSockets, keepAlive: this._keepAlive, proxy: { proxyAuth: `${proxyUrl.username}:${proxyUrl.password}`, host: proxyUrl.hostname, port: proxyUrl.port } } let tunnelAgent: Function const overHttps = proxyUrl.protocol === 'https:' if (usingSsl) { tunnelAgent = overHttps ? tunnel.httpsOverHttps : tunnel.httpsOverHttp } else { tunnelAgent = overHttps ? tunnel.httpOverHttps : tunnel.httpOverHttp } agent = tunnelAgent(agentOptions) this._proxyAgent = agent } // if reusing agent across request and tunneling agent isn't assigned create a new agent if (this._keepAlive && !agent) { const options = {keepAlive: this._keepAlive, maxSockets: maxSockets} agent = usingSsl ? new https.Agent(options) : new http.Agent(options) this._agent = agent } // if not using private agent and tunnel agent isn't setup then use global agent if (!agent) { agent = usingSsl ? https.globalAgent : http.globalAgent } if (usingSsl && this._ignoreSslError) { // we don't want to set NODE_TLS_REJECT_UNAUTHORIZED=0 since that will affect request for entire process // http.RequestOptions doesn't expose a way to modify RequestOptions.agent.options // we have to cast it to any and change it directly agent.options = Object.assign(agent.options || {}, { rejectUnauthorized: false }) } return agent } private _performExponentialBackoff(retryNumber: number): Promise { retryNumber = Math.min(ExponentialBackoffCeiling, retryNumber) const ms: number = ExponentialBackoffTimeSlice * Math.pow(2, retryNumber) return new Promise(resolve => setTimeout(() => resolve(), ms)) } private static dateTimeDeserializer(key: any, value: any): any { if (typeof value === 'string') { let a = new Date(value) if (!isNaN(a.valueOf())) { return a } } return value } private async _processResponse( res: ifm.IHttpClientResponse, options: ifm.IRequestOptions ): Promise> { return new Promise>(async (resolve, reject) => { const statusCode: number = res.message.statusCode const response: ifm.ITypedResponse = { statusCode: statusCode, result: null, headers: {} } // not found leads to null obj returned if (statusCode == HttpCodes.NotFound) { resolve(response) } let obj: any let contents: string // get the result from the body try { contents = await res.readBody() if (contents && contents.length > 0) { if (options && options.deserializeDates) { obj = JSON.parse(contents, HttpClient.dateTimeDeserializer) } else { obj = JSON.parse(contents) } response.result = obj } response.headers = res.message.headers } catch (err) { // Invalid resource (contents not json); leaving result obj null } // note that 3xx redirects are handled by the http layer. if (statusCode > 299) { let msg: string // if exception/error in body, attempt to get better error if (obj && obj.message) { msg = obj.message } else if (contents && contents.length > 0) { // it may be the case that the exception is in the body message as string msg = contents } else { msg = 'Failed request: (' + statusCode + ')' } let err: Error = new Error(msg) // attach statusCode and body obj (if available) to the error object err['statusCode'] = statusCode if (response.result) { err['result'] = response.result } reject(err) } else { resolve(response) } }) } }