Merge pull request #1991 from microsoft/feature/private-apis

feature/private apis

.vscode/launch.json

@@ -256,6 +256,20 @@
             "console": "internalConsole",
             "stopAtEntry": false
         },
+        {
+            "name": "Launch Login (github - device)",
+            "type": "coreclr",
+            "request": "launch",
+            "preLaunchTask": "build",
+            "program": "${workspaceFolder}/src/kiota/bin/Debug/net7.0/kiota.dll",
+            "args": ["login",
+                    "github",
+                    "device"
+                ],
+            "cwd": "${workspaceFolder}/src/kiota",
+            "console": "internalConsole",
+            "stopAtEntry": false
+        },
         {
             "name": ".NET Core Attach",
             "type": "coreclr",

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - Added support for GitHub based API search.[#1866](https://github.com/microsoft/kiota/issues/1866)
+- Added login/logout commands to access API descriptions in private GitHub repositories. [#1983](https://github.com/microsoft/kiota/issues/1983)
 - Added support for scalar request bodies Python [#1571](https://github.com/microsoft/kiota/issues/1571)
 - Sets property defaults in constructor and removes duplicate properties defined in base types from model serialization and deserialization methods in Python. [#1726](https://github.com/microsoft/kiota/issues/1726)
 - Added support for scalar request bodies in PHP [#1937](https://github.com/microsoft/kiota/pull/1937)

docs/using.md

@@ -19,6 +19,8 @@ Kiota offers the following commands to help you build your API client:
 - [generate](#client-generation): generate a client for any API from its description.
 - [update](#client-update): update existing clients from previous generations.
 - [info](#language-information): show languages and runtime dependencies information.
+- [login](#login): logs in to private API descriptions repositories.
+- [logout](#logout): logs out from private API description repositories.
 
 ## Description search
 
@@ -402,6 +404,45 @@ The generate command accepts optional parameters commonly available on the other
 - [--log-level](#--log-level---ll)
 - [--output](#--output--o)
 
+## Login
+
+Use kiota login to sign in to private repositories and search for/display/generate clients for private API descriptions. This command makes sub-commands available to sign in to specific authentication providers.
+
+### Login to GitHub
+
+```shell
+kiota login github <device|pat>
+      [(--log-level | --ll) <level>]
+      [(--pat) <patValue>]
+```
+
+Use `kiota login github device` to sign in using a device code, you will be prompted to sign-in using a web browser.
+
+Use `kiota login github pat --pat patValue` to sign in using a Personal Access Token you previously generated. You can use both classic PATs or granular PATs (beta). Classic PATs need the **repo** permission and to be granted access to the target organizations. Granular PATs need a **read-only** permission for the **contents** scope under the **repository** category and they need to be consented for all private repositories or selected private repositories.
+
+> Note: for more information about adding API descriptions to the GitHub index, see [Adding an API to search](add-api.md)
+
+### Optional parameters
+
+The generate command accepts optional parameters commonly available on the other commands:
+
+- [--log-level](#--log-level---ll)
+
+## Logout
+
+Use kiota logout to logout from a private repository containing API descriptions.
+
+```shell
+kiota logout github
+      [(--log-level | --ll) <level>]
+```
+
+### Optional parameters
+
+The generate command accepts optional parameters commonly available on the other commands:
+
+- [--log-level](#--log-level---ll)
+
 ## Common parameters
 
 The following parameters are available across multiple commands.

src/Kiota.Builder/Configuration/SearchConfiguration.cs

@@ -4,5 +4,12 @@ namespace Kiota.Builder.Configuration;
 
 public class SearchConfiguration : SearchConfigurationBase {
     public Uri APIsGuruListUrl { get; set; } = new ("https://raw.githubusercontent.com/APIs-guru/openapi-directory/gh-pages/v2/list.json");
-    public Uri GitHubBlockListUrl { get; set; } = new ("https://raw.githubusercontent.com/microsoft/kiota/main/resources/index-block-list.yml");
+    public GitHubConfiguration GitHub { get; set; } = new();
+}
+
+public class GitHubConfiguration {
+    public string AppId { get; set; } = "Iv1.9ed2bcb878c90617";
+    public Uri ApiBaseUrl { get; set; } = new ("https://api.github.com");
+    public Uri BlockListUrl { get; set; } = new ("https://raw.githubusercontent.com/microsoft/kiota/main/resources/index-block-list.yml");
+    public Uri AppManagement { get; set; } = new("https://aka.ms/kiota/install/github");
 }

src/Kiota.Builder/Configuration/SearchConfigurationBase.cs

@@ -1,7 +1,5 @@
 namespace Kiota.Builder.Configuration;
 
 public abstract class SearchConfigurationBase {
-    public string SearchTerm { get; set; }
     public bool ClearCache { get; set; }
-    public string Version { get; set; }
 }

src/Kiota.Builder/Extensions/UriExtensions.cs

@@ -0,0 +1,11 @@
+using System;
+using System.IO;
+
+namespace Kiota.Builder.Extensions;
+
+public static class UriExtensions {
+    public static string GetFileName(this Uri uri) {
+        if(uri is null) return string.Empty;
+        return Path.GetFileName($"{uri.Scheme}://{uri.Host}{uri.AbsolutePath}");
+    }
+}

src/Kiota.Builder/KiotaBuilder.cs

@@ -34,14 +34,19 @@ public class KiotaBuilder
 {
     private readonly ILogger<KiotaBuilder> logger;
     private readonly GenerationConfiguration config;
+    private readonly HttpClient httpClient;
     private OpenApiDocument originalDocument;
     private OpenApiDocument openApiDocument;
     internal void SetOpenApiDocument(OpenApiDocument document) => openApiDocument = document ?? throw new ArgumentNullException(nameof(document));
 
-    public KiotaBuilder(ILogger<KiotaBuilder> logger, GenerationConfiguration config)
+    public KiotaBuilder(ILogger<KiotaBuilder> logger, GenerationConfiguration config, HttpClient client)
     {
+        ArgumentNullException.ThrowIfNull(logger);
+        ArgumentNullException.ThrowIfNull(config);
+        ArgumentNullException.ThrowIfNull(client);
         this.logger = logger;
         this.config = config;
+        this.httpClient = client;
     }
     private void CleanOutputDirectory()
     {
@@ -222,12 +227,12 @@ private async Task<Stream> LoadStream(string inputPath, CancellationToken cancel
         Stream input;
         if (inputPath.StartsWith("http"))
             try {
-                using var httpClient = new HttpClient();
                 var cachingProvider = new DocumentCachingProvider(httpClient, logger) {
                     ClearCache = config.ClearCache,
                 };
-                var fileName = Path.GetFileName(inputPath) is string name && !string.IsNullOrEmpty(name) ? name : "description.yml";
-                input = await cachingProvider.GetDocumentAsync(new Uri(inputPath), "generation", fileName, cancellationToken: cancellationToken);
+                var targetUri = new Uri(inputPath);
+                var fileName = targetUri.GetFileName() is string name && !string.IsNullOrEmpty(name) ? name : "description.yml";
+                input = await cachingProvider.GetDocumentAsync(targetUri, "generation", fileName, cancellationToken: cancellationToken);
             } catch (HttpRequestException ex) {
                 throw new InvalidOperationException($"Could not download the file at {inputPath}, reason: {ex.Message}", ex);
             }

src/Kiota.Builder/KiotaSearcher.cs

@@ -10,39 +10,47 @@
 using Kiota.Builder.SearchProviders.GitHub;
 using Kiota.Builder.SearchProviders.MSGraph;
 using Microsoft.Extensions.Logging;
+using Microsoft.Kiota.Abstractions.Authentication;
 
 namespace Kiota.Builder;
 
 public class KiotaSearcher {
-    private readonly ILogger<KiotaSearcher> logger;
-    private readonly SearchConfiguration config;
-    public KiotaSearcher(ILogger<KiotaSearcher> logger, SearchConfiguration config) {
+    private readonly ILogger<KiotaSearcher> _logger;
+    private readonly SearchConfiguration _config;
+    private readonly HttpClient _httpClient;
+    private readonly IAuthenticationProvider _gitHubAuthenticationProvider;
+    private readonly Func<CancellationToken, Task<bool>> _isGitHubSignedInCallBack;
+
+    public KiotaSearcher(ILogger<KiotaSearcher> logger, SearchConfiguration config, HttpClient httpClient, IAuthenticationProvider gitHubAuthenticationProvider, Func<CancellationToken, Task<bool>> isGitHubSignedInCallBack) {
         ArgumentNullException.ThrowIfNull(logger);
         ArgumentNullException.ThrowIfNull(config);
-        this.logger = logger;
-        this.config = config;
+        ArgumentNullException.ThrowIfNull(httpClient);
+        _logger = logger;
+        _config = config;
+        _httpClient = httpClient;
+        _gitHubAuthenticationProvider = gitHubAuthenticationProvider;
+        _isGitHubSignedInCallBack = isGitHubSignedInCallBack;
     }
-    public async Task<IDictionary<string, SearchResult>> SearchAsync(CancellationToken cancellationToken) {
-        if (string.IsNullOrEmpty(config.SearchTerm)) {
-            logger.LogError("no search term provided");
+    public async Task<IDictionary<string, SearchResult>> SearchAsync(string searchTerm, string version, CancellationToken cancellationToken) {
+        if (string.IsNullOrEmpty(searchTerm)) {
+            _logger.LogError("no search term provided");
             return new Dictionary<string, SearchResult>();
         }
-        using var client = new HttpClient();
-        var apiGurusSearchProvider = new APIsGuruSearchProvider(config.APIsGuruListUrl, client, logger, config.ClearCache);
-        logger.LogDebug("searching for {searchTerm}", config.SearchTerm);
-        logger.LogDebug("searching APIs.guru with url {url}", config.APIsGuruListUrl);
+        var apiGurusSearchProvider = new APIsGuruSearchProvider(_config.APIsGuruListUrl, _httpClient, _logger, _config.ClearCache);
+        _logger.LogDebug("searching for {searchTerm}", searchTerm);
+        _logger.LogDebug("searching APIs.guru with url {url}", _config.APIsGuruListUrl);
         var oasProvider = new OpenApiSpecSearchProvider();
-        var githubProvider = new GitHubSearchProvider(client, config.GitHubBlockListUrl, logger, config.ClearCache);
+        var githubProvider = new GitHubSearchProvider(_httpClient, _logger, _config.ClearCache, _config.GitHub, _gitHubAuthenticationProvider, _isGitHubSignedInCallBack);
         var results = await Task.WhenAll(
-                        SearchProviderAsync(apiGurusSearchProvider, cancellationToken),
-                        SearchProviderAsync(oasProvider, cancellationToken),
-                        SearchProviderAsync(githubProvider, cancellationToken));
+                        SearchProviderAsync(searchTerm, version, apiGurusSearchProvider, cancellationToken),
+                        SearchProviderAsync(searchTerm, version, oasProvider, cancellationToken),
+                        SearchProviderAsync(searchTerm, version, githubProvider, cancellationToken));
         return results.SelectMany(static x => x)
                 .ToDictionary(static x => x.Key, static x => x.Value, StringComparer.OrdinalIgnoreCase);
     }
-    private async Task<IDictionary<string, SearchResult>> SearchProviderAsync(ISearchProvider provider, CancellationToken cancellationToken) {
+    private static async Task<IDictionary<string, SearchResult>> SearchProviderAsync(string searchTerm, string version, ISearchProvider provider, CancellationToken cancellationToken) {
         var providerPrefix = $"{provider.ProviderKey}{ProviderSeparator}";
-        var results = await provider.SearchAsync(config.SearchTerm.Replace(providerPrefix, string.Empty), config.Version, cancellationToken);
+        var results = await provider.SearchAsync(searchTerm.Replace(providerPrefix, string.Empty), version, cancellationToken);
 
         return results.Select(x => ($"{providerPrefix}{x.Key}", x.Value))
                     .ToDictionary(static x => x.Item1, static x => x.Value, StringComparer.OrdinalIgnoreCase);

src/Kiota.Builder/SearchProviders/GitHub/Authentication/AccessCodeResponse.cs

@@ -0,0 +1,19 @@
+using System;
+using System.Text.Json.Serialization;
+
+namespace Kiota.Builder.SearchProviders.GitHub.Authentication;
+
+public class AccessCodeResponse {
+	[JsonPropertyName("access_token")]
+	public string AccessToken { get; set; }
+	[JsonPropertyName("token_type")]
+	public string TokenType { get; set; }
+	[JsonPropertyName("scope")]
+	public string Scope { get; set; }
+	[JsonPropertyName("error")]
+	public string Error { get; set; }
+	[JsonPropertyName("error_description")]
+	public string ErrorDescription { get; set; }
+	[JsonPropertyName("error_uri")]
+	public Uri ErrorUri { get; set; }
+}

src/Kiota.Builder/SearchProviders/GitHub/Autentication/GitHubAnonymousAuthenticationProvider.cs → src/Kiota.Builder/SearchProviders/GitHub/Authentication/AnonymousAuthenticationProvider.cs

@@ -5,11 +5,12 @@
 using Microsoft.Kiota.Abstractions.Authentication;
 
 namespace Kiota.Builder.SearchProviders.GitHub.Authentication;
-public class GitHubAnonymousAuthenticationProvider : IAuthenticationProvider
+public class AnonymousAuthenticationProvider : IAuthenticationProvider
 {
-    public Task AuthenticateRequestAsync(RequestInformation request, Dictionary<string, object> additionalAuthenticationContext = null, CancellationToken cancellationToken = default)
+    public virtual Task AuthenticateRequestAsync(RequestInformation request, Dictionary<string, object> additionalAuthenticationContext = null, CancellationToken cancellationToken = default)
     {
         request.Headers.Add("User-Agent", $"Kiota/{GetType().Assembly.GetName().Version}");
+        request.Headers.Add("X-GitHub-Api-Version", "2022-11-28");
         return Task.CompletedTask;
     }
 }

src/Kiota.Builder/SearchProviders/GitHub/Authentication/BaseAuthenticationProvider.cs

@@ -0,0 +1,59 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Microsoft.Kiota.Abstractions;
+using Microsoft.Kiota.Abstractions.Authentication;
+
+namespace Kiota.Builder.SearchProviders.GitHub.Authentication;
+
+public class BaseAuthenticationProvider<T> : AnonymousAuthenticationProvider where T: class, IAccessTokenProvider
+{
+	public BaseAuthenticationProvider(string clientId,
+        string scope,
+        IEnumerable<string> validHosts,
+        ILogger logger,
+        Func<string, string, IEnumerable<string>, T> accessTokenProviderFactory,
+        bool enableCache = true)
+	{
+        ArgumentNullException.ThrowIfNull(validHosts);
+        ArgumentNullException.ThrowIfNull(logger);
+        ArgumentNullException.ThrowIfNull(accessTokenProviderFactory);
+        if (string.IsNullOrEmpty(clientId))
+			throw new ArgumentNullException(nameof(clientId));
+		if (string.IsNullOrEmpty(scope))
+			throw new ArgumentNullException(nameof(scope));
+
+        AccessTokenProvider = accessTokenProviderFactory(clientId, scope, validHosts);
+        if(enableCache)
+            AccessTokenProvider = new TempFolderCachingAccessTokenProvider {
+                Concrete = AccessTokenProvider,
+                Logger = logger,
+                ApiBaseUrl = new Uri($"https://{validHosts.FirstOrDefault() ?? "api.github.com"}"),
+                AppId = clientId,
+            };
+	}
+	public IAccessTokenProvider AccessTokenProvider {get; private set;}
+    private const string AuthorizationHeaderKey = "Authorization";
+    private const string ClaimsKey = "claims";
+	public override Task AuthenticateRequestAsync(RequestInformation request, Dictionary<string, object> additionalAuthenticationContext = null, CancellationToken cancellationToken = default) {
+		ArgumentNullException.ThrowIfNull(request);
+        return AuthenticateRequestInternalAsync(request, additionalAuthenticationContext, cancellationToken);
+    }
+	private async Task AuthenticateRequestInternalAsync(RequestInformation request, Dictionary<string, object> additionalAuthenticationContext = null, CancellationToken cancellationToken = default) {
+        await base.AuthenticateRequestAsync(request, additionalAuthenticationContext, cancellationToken).ConfigureAwait(false);
+        if(additionalAuthenticationContext != null &&
+            additionalAuthenticationContext.ContainsKey(ClaimsKey) &&
+            request.Headers.ContainsKey(AuthorizationHeaderKey))
+            request.Headers.Remove(AuthorizationHeaderKey);
+
+        if(!request.Headers.ContainsKey(AuthorizationHeaderKey))
+        {
+            var token = await AccessTokenProvider.GetAuthorizationTokenAsync(request.URI, additionalAuthenticationContext, cancellationToken);
+            if(!string.IsNullOrEmpty(token))
+                request.Headers.Add(AuthorizationHeaderKey, $"Bearer {token}");
+        }
+	}
+}

src/Kiota.Builder/SearchProviders/GitHub/Authentication/ITokenStorageService.cs

@@ -0,0 +1,11 @@
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Kiota.Builder.SearchProviders.GitHub.Authentication;
+
+public interface ITokenStorageService {
+    Task<string> GetTokenAsync(CancellationToken cancellationToken);
+    Task SetTokenAsync(string value, CancellationToken cancellationToken);
+    Task<bool> IsTokenPresentAsync(CancellationToken cancellationToken);
+    Task<bool> DeleteTokenAsync(CancellationToken cancellationToken);
+}

src/Kiota.Builder/SearchProviders/GitHub/Authentication/PatAccessTokenProvider.cs

@@ -0,0 +1,20 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Kiota.Abstractions.Authentication;
+
+namespace Kiota.Builder.SearchProviders.GitHub.Authentication;
+
+public class PatAccessTokenProvider : IAccessTokenProvider
+{
+    public AllowedHostsValidator AllowedHostsValidator { get; set; } = new();
+    public required ITokenStorageService StorageService { get; init; }
+    public Task<string> GetAuthorizationTokenAsync(Uri uri, Dictionary<string, object> additionalAuthenticationContext = null, CancellationToken cancellationToken = default)
+    {
+        ArgumentNullException.ThrowIfNull(uri);
+        if("https".Equals(uri.Scheme, StringComparison.OrdinalIgnoreCase) && AllowedHostsValidator.IsUrlHostValid(uri))
+            return StorageService.GetTokenAsync(cancellationToken);
+        return Task.FromResult(string.Empty);
+    }
+}

src/Kiota.Builder/SearchProviders/GitHub/Authentication/PatAuthenticationProvider.cs

@@ -0,0 +1,17 @@
+
+using System;
+using System.Collections.Generic;
+using Microsoft.Extensions.Logging;
+
+namespace Kiota.Builder.SearchProviders.GitHub.Authentication;
+public class PatAuthenticationProvider : BaseAuthenticationProvider<PatAccessTokenProvider>
+{
+    public PatAuthenticationProvider(string clientId, string scope, IEnumerable<string> validHosts, ILogger logger, ITokenStorageService StorageService) :
+        base(clientId, scope, validHosts, logger, (_, _, validHosts) => new PatAccessTokenProvider {
+            StorageService = StorageService,
+            AllowedHostsValidator = new (validHosts),
+        }, false)
+    {
+        ArgumentNullException.ThrowIfNull(StorageService);
+    }
+}