diff --git a/extension.yaml b/src/openhound_github/extension.yaml similarity index 100% rename from extension.yaml rename to src/openhound_github/extension.yaml diff --git a/src/openhound_github/graphql.py b/src/openhound_github/graphql.py index a485f57..8bdfb1a 100644 --- a/src/openhound_github/graphql.py +++ b/src/openhound_github/graphql.py @@ -118,17 +118,9 @@ ENTERPRISE_SAML_QUERY = """ query EnterpriseSAML($slug: String!, $count: Int = 100, $after: String = null) { enterprise(slug: $slug) { - id - name slug ownerInfo { samlIdentityProvider { - id - issuer - ssoUrl - digestMethod - signatureMethod - idpCertificate externalIdentities(first: $count, after: $after) { totalCount nodes { @@ -141,14 +133,14 @@ username } scimIdentity { - username - givenName - familyName emails { - value primary type + value } + familyName + givenName + username } user { id @@ -166,6 +158,26 @@ } """ +ENTERPRISE_SAML_PROVIDER_QUERY = """ +query EnterpriseSAMLProvider($slug: String!) { + enterprise(slug: $slug) { + id + name + slug + ownerInfo { + samlIdentityProvider { + id + issuer + ssoUrl + digestMethod + signatureMethod + idpCertificate + } + } + } +} +""" + TEAMS_QUERY = """ query Teams($login: String!, $count: Int!, $after: String) { organization(login: $login) { @@ -307,56 +319,18 @@ """ SAML_QUERY = """ -query SAML($login: String!, $count: Int = 100, $after: String = null) { +query SAMLProvider($login: String!) { organization(login: $login) { id name login samlIdentityProvider { - digestMethod - externalIdentities(first: $count, after: $after) { - nodes { - guid - id - samlIdentity { - attributes { - metadata - name - value - } - familyName - givenName - groups - nameId - username - } - scimIdentity { - emails { - primary - type - value - } - familyName - givenName - groups - username - } - user { - id - login - } - } - pageInfo { - endCursor - hasNextPage - } - totalCount - } id - idpCertificate issuer - signatureMethod ssoUrl + digestMethod + signatureMethod + idpCertificate } } } @@ -369,20 +343,13 @@ name login samlIdentityProvider { - digestMethod externalIdentities(first: $count, after: $after) { nodes { guid id samlIdentity { - attributes { - metadata - name - value - } familyName givenName - groups nameId username } @@ -394,7 +361,6 @@ } familyName givenName - groups username } user { @@ -408,11 +374,6 @@ } totalCount } - id - idpCertificate - issuer - signatureMethod - ssoUrl } } } diff --git a/src/openhound_github/helpers.py b/src/openhound_github/helpers.py index 78a7c4b..36bd036 100644 --- a/src/openhound_github/helpers.py +++ b/src/openhound_github/helpers.py @@ -4,15 +4,12 @@ from dlt.common import jsonpath from dlt.sources.helpers import requests +from dlt.sources.helpers.rest_client.auth import AuthConfigBase from dlt.sources.helpers.rest_client.paginators import ( JSONResponseCursorPaginator, ) from requests import Request -from openhound_github.auth import ( - GitHubAppInstallationAuth, -) - logger = logging.getLogger(__name__) @@ -154,7 +151,7 @@ def _has_graphql_errors(response: requests.Response) -> bool: return isinstance(response_data, dict) and bool(response_data.get("errors")) -def github_retry_policy(auth: GitHubAppInstallationAuth): +def github_retry_policy(auth: AuthConfigBase): def retry_policy( response: Optional[requests.Response], exception: Optional[BaseException] ) -> bool: diff --git a/src/openhound_github/kinds/edges.py b/src/openhound_github/kinds/edges.py index 4423f76..8d27862 100644 --- a/src/openhound_github/kinds/edges.py +++ b/src/openhound_github/kinds/edges.py @@ -49,9 +49,6 @@ # Other HAS_SECRET_SCANNING_ALERT = "GH_HasSecretScanningAlert" VALID_TOKEN = "GH_ValidToken" -HAS_WORKFLOW = "GH_HasWorkflow" -HAS_JOB = "GH_HasJob" -HAS_STEP = "GH_HasStep" DEPENDS_ON = "GH_DependsOn" DEPLOYS_TO = "GH_DeploysTo" CALLS_WORKFLOW = "GH_CallsWorkflow" @@ -143,3 +140,15 @@ CAN_PWN_REQUEST = "GH_CanPwnRequest" CAN_READ_SECRET = "GH_CanReadSecret" CAN_READ_SECRET_SCANNING_ALERT = "GH_CanReadSecretScanningAlert" + +# SAML edges +SAML_IMPLEMENTS = "SAML_Implements" +SAML_ISSUES_AS = "SAML_IssuesAs" +SAML_TRUSTS_ISSUER = "SAML_TrustsIssuer" +SAML_ISSUES_ASSERTIONS_TO = "SAML_IssuesAssertionsTo" +SAML_HAS_ASSERTION_CONSUMER_SERVICE = "SAML_HasAssertionConsumerService" +SAML_HAS_CLAIM_MAPPING = "SAML_HasClaimMapping" +SAML_HAS_ACCOUNT = "SAML_HasAccount" +SAML_ELIGIBLE_FOR = "SAML_EligibleFor" +SAML_ISSUES_ASSERTIONS_FOR = "SAML_IssuesAssertionsFor" +SAML_CAN_SIGN_IN_AS = "SAML_CanSignInAs" diff --git a/src/openhound_github/kinds/nodes.py b/src/openhound_github/kinds/nodes.py index ce75797..3f56bc0 100644 --- a/src/openhound_github/kinds/nodes.py +++ b/src/openhound_github/kinds/nodes.py @@ -58,8 +58,14 @@ WORKFLOW_JOB = "GH_WorkflowJob" WORKFLOW_STEP = "GH_WorkflowStep" - # Enterprise nodes ENTERPRISE = "GH_Enterprise" ENTERPRISE_TEAM = "GH_EnterpriseTeam" ENTERPRISE_ROLE = "GH_EnterpriseRole" + +# SAML nodes +SAML_FEDERATION_PROVIDER = "SAML_FederationProvider" +SAML_SERVICE_PROVIDER = "SAML_ServiceProvider" +SAML_ISSUER = "SAML_Issuer" +SAML_ASSERTION_CONSUMER_SERVICE = "SAML_AssertionConsumerService" +SAML_CLAIM_MAPPING = "SAML_ClaimMapping" diff --git a/src/openhound_github/lookup.py b/src/openhound_github/lookup.py index cc296dc..01afdd5 100644 --- a/src/openhound_github/lookup.py +++ b/src/openhound_github/lookup.py @@ -98,10 +98,10 @@ def idp(self) -> list: ) @lru_cache - def idp_for_org(self, org_login: str) -> list: + def idp_for_environment(self, environment_slug: str) -> list: return self._find_all_objects( - f"""SELECT id, issuer, sso_url FROM {self.schema}.saml_provider WHERE org_login = ?""", - [org_login], + f"""SELECT id, issuer, sso_url, environment_node_id, environment_name, environment_type FROM {self.schema}.saml_provider WHERE environment_slug = ?""", + [environment_slug], ) @lru_cache diff --git a/src/openhound_github/models/__init__.py b/src/openhound_github/models/__init__.py index 1476233..196970a 100644 --- a/src/openhound_github/models/__init__.py +++ b/src/openhound_github/models/__init__.py @@ -6,7 +6,6 @@ from .branch_push_allowance import BranchPushAllowance from .enterprise import Enterprise from .enterprise_admin import EnterpriseAdmin -from .enterprise_external_identity import EnterpriseExternalIdentity from .enterprise_helpers import enterprise_role_node_id, enterprise_team_node_id from .enterprise_managed_user import EnterpriseManagedUser from .enterprise_member import BaseUser, flatten_enterprise_member @@ -14,7 +13,6 @@ from .enterprise_role import EnterpriseRole from .enterprise_role_team import EnterpriseRoleTeam from .enterprise_role_user import EnterpriseRoleUser -from .enterprise_saml_provider import EnterpriseSamlProvider from .enterprise_team import EnterpriseTeam from .enterprise_team_member import EnterpriseTeamMember from .enterprise_team_organization import EnterpriseTeamOrganization @@ -41,7 +39,10 @@ from .repository_secret import RepoSecret from .repository_variable import RepoVariable from .runner import OrgRunner, OrgRunnerGroupMembership, RepoRunner, RunnerGroup +from .saml_assertion_consumer_service import SamlAssertionConsumerService from .saml_provider import SamlProvider +from .saml_service_provider import SamlServiceProvider +from .saml_issuer import SamlIssuer from .scim_user import ScimResource from .secret_scanning_alert import SecretScanningAlert from .team import Team @@ -92,7 +93,10 @@ "ProjectedEnterpriseTeam", "SelectedOrgSecret", "PersonalAccessTokenRequest", + "SamlAssertionConsumerService", "SamlProvider", + "SamlServiceProvider", + "SamlIssuer", "ExternalIdentity", "ScimResource", "RepoRoleAssignment", @@ -102,13 +106,11 @@ "EnvironmentBranchPolicy", "Enterprise", "EnterpriseAdmin", - "EnterpriseExternalIdentity", "EnterpriseManagedUser", "EnterpriseOrganization", "EnterpriseRole", "EnterpriseRoleTeam", "EnterpriseRoleUser", - "EnterpriseSamlProvider", "EnterpriseTeam", "EnterpriseTeamMember", "EnterpriseTeamOrganization", diff --git a/src/openhound_github/models/enterprise_external_identity.py b/src/openhound_github/models/enterprise_external_identity.py deleted file mode 100644 index 52120f2..0000000 --- a/src/openhound_github/models/enterprise_external_identity.py +++ /dev/null @@ -1,202 +0,0 @@ -from dataclasses import dataclass - -from openhound.core.asset import BaseAsset, EdgeDef, NodeDef -from openhound.core.models.entries_dataclass import ( - ConditionalEdgePath, - Edge, - EdgePath, - EdgeProperties, - PropertyMatch, -) -from pydantic import BaseModel, ConfigDict, Field - -from openhound_github.graph import GHNode, GHNodeProperties -from openhound_github.kinds import edges as ek -from openhound_github.kinds import nodes as nk -from openhound_github.main import app -from openhound_github.models.enterprise_saml_provider import EnterpriseSamlProvider - - -class EnterpriseIdentityName(BaseModel): - model_config = ConfigDict(extra="allow", populate_by_name=True) - - family_name: str | None = Field(alias="familyName", default=None) - given_name: str | None = Field(alias="givenName", default=None) - name_id: str | None = Field(alias="nameId", default=None) - username: str | None = None - - -class EnterpriseIdentityUser(BaseModel): - id: str - login: str - - -@dataclass -class GHEnterpriseExternalIdentityProperties(GHNodeProperties): - """Properties for an enterprise external identity. - - Attributes: - guid: The external identity GUID. - saml_identity_username: The SAML username. - saml_identity_name_id: The SAML NameID. - saml_identity_given_name: The SAML given name. - saml_identity_family_name: The SAML family name. - scim_identity_username: The SCIM username. - scim_identity_given_name: The SCIM given name. - scim_identity_family_name: The SCIM family name. - github_username: The mapped GitHub username. - github_user_id: The mapped GitHub user node ID. - environment_name: The enterprise environment name. - query_mapped_users: Query for mapped users. - """ - - guid: str | None = None - saml_identity_username: str | None = None - saml_identity_name_id: str | None = None - saml_identity_given_name: str | None = None - saml_identity_family_name: str | None = None - scim_identity_username: str | None = None - scim_identity_given_name: str | None = None - scim_identity_family_name: str | None = None - github_username: str | None = None - github_user_id: str | None = None - environment_name: str | None = None - query_mapped_users: str | None = None - - -@app.asset( - node=NodeDef( - kind=nk.EXTERNAL_IDENTITY, - description="GitHub Enterprise External Identity", - icon="arrows-left-right", - properties=GHEnterpriseExternalIdentityProperties, - ), - edges=[ - EdgeDef( - start=nk.SAML_IDENTITY_PROVIDER, - end=nk.EXTERNAL_IDENTITY, - kind=ek.HAS_EXTERNAL_IDENTITY, - description="Enterprise IdP has external identity", - traversable=False, - ), - EdgeDef( - start=nk.EXTERNAL_IDENTITY, - end=nk.USER, - kind=ek.MAPS_TO_USER, - description="External identity maps to GitHub user", - traversable=False, - ), - ], -) -class EnterpriseExternalIdentity(BaseAsset): - model_config = ConfigDict(extra="allow", populate_by_name=True) - - guid: str | None = None - id: str - saml_identity: EnterpriseIdentityName | None = Field( - alias="samlIdentity", default=None - ) - scim_identity: EnterpriseIdentityName | None = Field( - alias="scimIdentity", default=None - ) - user: EnterpriseIdentityUser | None = None - saml_provider_id: str - saml_provider_issuer: str | None = None - saml_provider_sso_url: str | None = None - enterprise_node_id: str - enterprise_slug: str - - @property - def node_id(self) -> str: - return self.id - - @property - def foreign_user(self) -> tuple[str | None, str | None]: - return EnterpriseSamlProvider.detect_foreign_environment( - self.saml_provider_issuer, self.saml_provider_sso_url - ) - - @property - def foreign_username(self) -> str | None: - if self.saml_identity and self.saml_identity.username: - return self.saml_identity.username - if self.scim_identity and self.scim_identity.username: - return self.scim_identity.username - return None - - @property - def as_node(self) -> GHNode: - return GHNode( - kinds=[nk.EXTERNAL_IDENTITY], - properties=GHEnterpriseExternalIdentityProperties( - name=self.guid or self.node_id, - displayname=self.guid or self.node_id, - node_id=self.node_id, - environmentid=self._lookup.enterprise_id(), - environment_name=self.enterprise_slug, - guid=self.guid, - saml_identity_username=self.saml_identity.username - if self.saml_identity - else None, - saml_identity_name_id=self.saml_identity.name_id - if self.saml_identity - else None, - saml_identity_given_name=self.saml_identity.given_name - if self.saml_identity - else None, - saml_identity_family_name=self.saml_identity.family_name - if self.saml_identity - else None, - scim_identity_username=self.scim_identity.username - if self.scim_identity - else None, - scim_identity_given_name=self.scim_identity.given_name - if self.scim_identity - else None, - scim_identity_family_name=self.scim_identity.family_name - if self.scim_identity - else None, - github_username=self.user.login if self.user else None, - github_user_id=self.user.id if self.user else None, - query_mapped_users=f"MATCH p=(:GH_ExternalIdentity {{node_id:'{self.node_id}'}})-[:GH_MapsToUser]->() RETURN p", - ), - ) - - @property - def edges(self): - yield Edge( - kind=ek.HAS_EXTERNAL_IDENTITY, - start=EdgePath(value=self.saml_provider_id, match_by="id"), - end=EdgePath(value=self.node_id, match_by="id"), - properties=EdgeProperties(traversable=False), - ) - if self.user and self.user.id: - yield Edge( - kind=ek.MAPS_TO_USER, - start=EdgePath(value=self.node_id, match_by="id"), - end=EdgePath(value=self.user.id, match_by="id"), - properties=EdgeProperties(traversable=False), - ) - - foreign_kind, _ = self.foreign_user - if foreign_kind and self.foreign_username: - match = PropertyMatch(key="name", value=self.foreign_username.upper()) - yield Edge( - kind=ek.MAPS_TO_USER, - start=EdgePath(value=self.node_id, match_by="id"), - end=ConditionalEdgePath( - kind=foreign_kind, - property_matchers=[match], - ), - properties=EdgeProperties(traversable=False), - ) - if self.user and self.user.id: - yield Edge( - kind=ek.SYNCED_TO_GH_USER, - start=ConditionalEdgePath( - kind=foreign_kind, - property_matchers=[match], - ), - end=EdgePath(value=self.user.id, match_by="id"), - properties=EdgeProperties(traversable=True), - ) diff --git a/src/openhound_github/models/enterprise_saml_provider.py b/src/openhound_github/models/enterprise_saml_provider.py deleted file mode 100644 index 4a30ed0..0000000 --- a/src/openhound_github/models/enterprise_saml_provider.py +++ /dev/null @@ -1,122 +0,0 @@ -from dataclasses import dataclass -from typing import ClassVar - -from dlt.common.libs.pydantic import DltConfig -from openhound.core.asset import BaseAsset, EdgeDef, NodeDef -from openhound.core.models.entries_dataclass import Edge, EdgePath, EdgeProperties -from pydantic import ConfigDict, Field - -from openhound_github.graph import GHNode, GHNodeProperties -from openhound_github.kinds import edges as ek -from openhound_github.kinds import nodes as nk -from openhound_github.main import app - - -@dataclass -class GHEnterpriseSamlProviderProperties(GHNodeProperties): - """Properties for an enterprise SAML identity provider. - - Attributes: - issuer: The SAML issuer. - sso_url: The SAML SSO URL. - signature_method: The SAML signature method. - digest_method: The SAML digest method. - idp_certificate: The IdP certificate. - foreign_environment_id: The correlated foreign environment ID. - environment_name: The enterprise environment name. - query_environments: Query for owning environments. - query_external_identities: Query for external identities. - """ - - issuer: str | None = None - sso_url: str | None = None - signature_method: str | None = None - digest_method: str | None = None - idp_certificate: str | None = None - foreign_environment_id: str | None = None - environment_name: str | None = None - query_environments: str | None = None - query_external_identities: str | None = None - - -@app.asset( - node=NodeDef( - kind=nk.SAML_IDENTITY_PROVIDER, - description="GitHub Enterprise SAML Identity Provider", - icon="id-badge", - properties=GHEnterpriseSamlProviderProperties, - ), - edges=[ - EdgeDef( - start=nk.ENTERPRISE, - end=nk.SAML_IDENTITY_PROVIDER, - kind=ek.HAS_SAML_IDENTITY_PROVIDER, - description="Enterprise uses this SAML IdP", - traversable=False, - ), - ], -) -class EnterpriseSamlProvider(BaseAsset): - model_config = ConfigDict(extra="allow", populate_by_name=True) - - id: str - issuer: str | None = None - sso_url: str | None = Field(alias="ssoUrl", default=None) - digest_method: str | None = Field(alias="digestMethod", default=None) - signature_method: str | None = Field(alias="signatureMethod", default=None) - idp_certificate: str | None = Field(alias="idpCertificate", default=None) - enterprise_node_id: str - enterprise_slug: str - - dlt_config: ClassVar[DltConfig] = {"return_validated_models": True} - - @property - def node_id(self) -> str: - return self.id - - @staticmethod - def detect_foreign_environment( - issuer: str | None, sso_url: str | None - ) -> tuple[str | None, str | None]: - if not issuer: - return None, None - if issuer.startswith("https://auth.pingone.com/"): - return "PingOneUser", issuer.split("/")[3] - if issuer.startswith("https://sts.windows.net/"): - return "AZUser", issuer.split("/")[3] - if issuer.startswith("http://www.okta.com/"): - return "Okta_User", sso_url.split("/")[2] if sso_url else None - return None, None - - @property - def as_node(self) -> GHNode: - _, foreign_environment_id = self.detect_foreign_environment( - self.issuer, self.sso_url - ) - return GHNode( - kinds=[nk.SAML_IDENTITY_PROVIDER], - properties=GHEnterpriseSamlProviderProperties( - name=self.node_id, - displayname=self.enterprise_slug, - node_id=self.node_id, - environmentid=self._lookup.enterprise_id(), - environment_name=self.enterprise_slug, - issuer=self.issuer, - sso_url=self.sso_url, - signature_method=self.signature_method, - digest_method=self.digest_method, - idp_certificate=self.idp_certificate, - foreign_environment_id=foreign_environment_id, - query_environments=f"MATCH p=(:GH_SamlIdentityProvider {{node_id:'{self.node_id}'}})<-[:GH_HasSamlIdentityProvider]-(:GH_Enterprise) RETURN p", - query_external_identities=f"MATCH p=(:GH_SamlIdentityProvider {{node_id:'{self.node_id}'}})-[:GH_HasExternalIdentity]->() RETURN p", - ), - ) - - @property - def edges(self): - yield Edge( - kind=ek.HAS_SAML_IDENTITY_PROVIDER, - start=EdgePath(value=self._lookup.enterprise_id(), match_by="id"), - end=EdgePath(value=self.node_id, match_by="id"), - properties=EdgeProperties(traversable=False), - ) diff --git a/src/openhound_github/models/external_identity.py b/src/openhound_github/models/external_identity.py index 959eec1..6f9f262 100644 --- a/src/openhound_github/models/external_identity.py +++ b/src/openhound_github/models/external_identity.py @@ -15,12 +15,16 @@ from openhound_github.kinds import nodes as nk from openhound_github.main import app -_FOREIGN_USER_KIND: dict[str, str] = { - "entra": "AZUser", - "okta": "OktaUser", - "pingone": "PingOneUser", -} +from .saml_helpers import ( + build_service_provider_node_id, + detect_foreign_idp, + foreign_user_kind, +) +@dataclass +class SAMLHasAccountEdgeProperties(EdgeProperties): + match_values: list[str] | None = None + account_state: str = "unknown" @dataclass class GHExternalIdentityProperties(GHNodeProperties): @@ -37,7 +41,7 @@ class GHExternalIdentityProperties(GHNodeProperties): scim_identity_family_name: The family name from the SCIM identity. github_username: The GitHub login of the linked user. github_user_id: The GraphQL ID of the linked GitHub user. - environment_name: The name of the environment (GitHub organization). + environment_name: The name of the environment (GitHub organization or enterprise). query_mapped_users: Query for mapped users. """ @@ -56,13 +60,13 @@ class GHExternalIdentityProperties(GHNodeProperties): class SCIMIdentity(BaseModel): - family_name: str | None = Field(alias="FamilyName", default=None) + family_name: str | None = Field(alias="familyName", default=None) given_name: str | None = Field(alias="givenName", default=None) username: str | None = None class SAMLIdentity(BaseModel): - family_name: str | None = Field(alias="FamilyName", default=None) + family_name: str | None = Field(alias="familyName", default=None) given_name: str | None = Field(alias="givenName", default=None) name_id: str | None = Field(alias="nameId", default=None) username: str | None = None @@ -102,6 +106,13 @@ class User(BaseModel): description="Foreign IdP user is synced to a GitHub user", traversable=True, ), + EdgeDef( + start=nk.SAML_SERVICE_PROVIDER, + end=nk.USER, + kind=ek.SAML_HAS_ACCOUNT, + description="Normalized SAML service provider has this downstream GitHub account", + traversable=False, + ), ], ) class ExternalIdentity(BaseAsset): @@ -111,16 +122,12 @@ class ExternalIdentity(BaseAsset): guid: str id: str - saml_identity: SAMLIdentity = Field(alias="samlIdentity") - scim_identity: SCIMIdentity | None = Field(alias="scimIdentity") + saml_identity: SAMLIdentity | None = Field(alias="samlIdentity", default=None) + scim_identity: SCIMIdentity | None = Field(alias="scimIdentity", default=None) user: User | None = None # Additional - org_login: str - - @property - def org_node_id(self) -> str | None: - return self._lookup.org_id_for_login(self.org_login) + environment_slug: str @property def node_id(self) -> str: @@ -159,87 +166,128 @@ def as_node(self) -> GHNode: else None, github_username=self.user.login if self.user else None, github_user_id=self.user.id if self.user else None, - environment_name=self.org_login, - environmentid=self.org_node_id, - query_mapped_users=f"MATCH p=(:GH_ExternalIdentity {{node_id:'{self.node_id.upper()}'}})-[:GH_MapsToUser]->() RETURN p", + environment_name=self.idp["environment_name"] if self.idp else None, + environmentid=self.idp["environment_node_id"] if self.idp else None, + query_mapped_users=f"MATCH p=(:GH_ExternalIdentity {{node_id:'{self.node_id}'}})-[:GH_MapsToUser]->() RETURN p", ), ) - @staticmethod - def detect_foreign_idp( - issuer: str | None, sso_url: str | None - ) -> tuple[str | None, str | None]: - """Detect the foreign IdP type and tenant/environment ID from the issuer or SSO URL.""" - if not issuer: - return None, None - if issuer.startswith("https://auth.pingone.com/"): - return "pingone", issuer.split("/")[3] - if issuer.startswith("https://sts.windows.net/"): - return "entra", issuer.split("/")[3] - if issuer.startswith("http://www.okta.com/"): - domain = (sso_url or "").split("/")[2] if sso_url else None - return "okta", domain - return None, None - @property def idp(self) -> dict: - ext_idp = self._lookup.idp_for_org(self.org_login) + ext_idp = self._lookup.idp_for_environment(self.environment_slug) if not ext_idp: - return {"id": None, "issuer": None, "sso_url": None} - id, issuer, sso_url = ext_idp[0] + return { + "id": None, + "issuer": None, + "sso_url": None, + "environment_node_id": None, + "environment_name": None, + } + provider_id, issuer, sso_url, environment_node_id, environment_name, environment_type = ext_idp[0] return { - "id": id, + "id": provider_id, "issuer": issuer, "sso_url": sso_url, + "environment_node_id": environment_node_id, + "environment_name": environment_name, + "environment_type": environment_type, } @property def _maps_to_user_edges(self): - if self.saml_identity: - foreign_idp_type, foreign_env_id = self.detect_foreign_idp( - issuer=self.idp["issuer"], - sso_url=self.idp["sso_url"], + foreign_idp_type, foreign_env_id = detect_foreign_idp( + issuer=self.idp["issuer"], + sso_url=self.idp["sso_url"], + ) + foreign_kind = foreign_user_kind(foreign_idp_type) + + foreign_env_key = None + if foreign_idp_type == "okta": + foreign_env_key = "tenant_domain" + elif foreign_idp_type == "pingone": + foreign_env_key = "environmentid" + elif foreign_idp_type == "entra": + foreign_env_key = "tenantid" + + foreign_username = None + if self.saml_identity and self.saml_identity.name_id: + foreign_username = self.saml_identity.name_id + elif self.saml_identity and self.saml_identity.username: + foreign_username = self.saml_identity.username + elif self.scim_identity and self.scim_identity.username: + foreign_username = self.scim_identity.username + + match_key = "name" + if foreign_idp_type == "pingone": + match_key = "email" + elif foreign_idp_type == "okta": + match_key = "login" + + # # GH_MapsToUser → foreign IdP user node (match by name) + matchers = None + if foreign_kind and foreign_username: + matchers = [PropertyMatch(key=match_key, value=foreign_username)] + if foreign_env_key and foreign_env_id: + matchers.append( + PropertyMatch(key=foreign_env_key, value=foreign_env_id) + ) + + yield Edge( + kind=ek.MAPS_TO_USER, + start=EdgePath(value=self.node_id, match_by="id"), + end=ConditionalEdgePath( + kind=foreign_kind, + property_matchers=matchers + ), + properties=EdgeProperties(traversable=False), + ) + + # SyncedToGHUser: foreign IdP user → GitHub user (traversable, with composition) + if matchers and self.user and self.user.id: + gh_id = self.node_id.upper() + q = ( + f"MATCH p=()<-[:GH_SyncedToEnvironment]-(:GH_SamlIdentityProvider)" + f"-[:GH_HasExternalIdentity]->(:GH_ExternalIdentity)" + f"-[:GH_MapsToUser]->(n) " + f"WHERE n.objectid = '{gh_id}' OR n.name = '{foreign_username.upper()}' RETURN p" ) - foreign_kind = _FOREIGN_USER_KIND.get(foreign_idp_type or "", "") - foreign_username = ( - self.saml_identity.username or self.scim_identity.username + yield Edge( + kind=ek.SYNCED_TO_GH_USER, + start=ConditionalEdgePath( + kind=foreign_kind, + property_matchers=matchers + ), + end=EdgePath(value=self.user.id, match_by="id"), + properties=GHEdgeProperties( + traversable=True, + composed=True, + query_composition=q, + ), ) - # # GH_MapsToUser → foreign IdP user node (match by name) - if foreign_kind and foreign_username: - match_with = PropertyMatch(key="name", value=foreign_username.upper()) - yield Edge( - kind=ek.MAPS_TO_USER, - start=EdgePath(value=self.node_id, match_by="id"), - end=ConditionalEdgePath( - kind="User", property_matchers=[match_with] - ), - properties=EdgeProperties(traversable=False), - ) + @property + def service_provider_node_id(self) -> str | None: + return build_service_provider_node_id( + self.idp.get("environment_type"), + self.environment_slug, + ) + + @property + def saml_match_values(self) -> list[str]: + values = [] - # SyncedToGHUser: foreign IdP user → GitHub user (traversable, with composition) - if foreign_kind and foreign_username and self.node_id: - match_with = PropertyMatch(key="name", value=foreign_username.upper()) + if self.saml_identity and self.saml_identity.name_id: + values.append(self.saml_identity.name_id) - gh_id = self.node_id.upper() - q = ( - f"MATCH p=()<-[:GH_SyncedToEnvironment]-(:GH_SamlIdentityProvider)" - f"-[:GH_HasExternalIdentity]->(:GH_ExternalIdentity)" - f"-[:GH_MapsToUser]->(n) " - f"WHERE n.objectid = '{gh_id}' OR n.name = '{foreign_username.upper()}' RETURN p" - ) - yield Edge( - kind=ek.SYNCED_TO_GH_USER, - start=ConditionalEdgePath( - kind="User", property_matchers=[match_with] - ), - end=EdgePath(value=self.node_id, match_by="id"), - properties=GHEdgeProperties( - traversable=True, - composed=True, - query_composition=q, - ), - ) + cleaned = [] + seen = set() + for value in values: + v = str(value).strip() + if v and v not in seen: + cleaned.append(v) + seen.add(v) + + return cleaned @property def edges(self): @@ -250,6 +298,22 @@ def edges(self): properties=EdgeProperties(traversable=False), ) + has_idp = bool(self.idp.get("id")) + service_provider_node_id = self.service_provider_node_id + match_values = self.saml_match_values + + if has_idp and service_provider_node_id and self.user and self.user.id and match_values: + yield Edge( + kind=ek.SAML_HAS_ACCOUNT, + start=EdgePath(value=service_provider_node_id, match_by="id"), + end=EdgePath(value=self.user.id, match_by="id"), + properties=SAMLHasAccountEdgeProperties( + traversable=False, + match_values=match_values, + account_state="unknown", + ), + ) + # GH_MapsToUser → linked GitHub user node (match by id) if self.user and self.user.id: yield Edge( diff --git a/src/openhound_github/models/repository.py b/src/openhound_github/models/repository.py index c217eb6..6fc30e2 100644 --- a/src/openhound_github/models/repository.py +++ b/src/openhound_github/models/repository.py @@ -252,7 +252,7 @@ def as_node(self) -> GHNode: query_branch_protection_rules=f"MATCH p=(:GH_Repository {{node_id: '{rid}'}})-[:GH_Contains]->(:GH_BranchProtectionRule) RETURN p", query_roles=f"MATCH p=(:GH_RepoRole)-[*1..]->(:GH_Repository {{node_id: '{rid}'}}) RETURN p", query_teams=f"MATCH p=(:GH_Team)-[:GH_MemberOf|GH_HasRole*1..]->(:GH_RepoRole)-[]->(:GH_Repository {{node_id: '{rid}'}}) RETURN p", - query_workflows=f"MATCH p=(:GH_Repository {{node_id:'{rid}'}})-[:GH_HasWorkflow]->(:GH_Workflow)-[:GH_HasJob]->(:GH_WorkflowJob)-[:GH_HasStep]->(step:GH_WorkflowStep) OPTIONAL MATCH p1=(step)-[:GH_UsesSecret]->(:GH_Secret) OPTIONAL MATCH p2=(step)-[:GH_UsesVariable]->(:GH_Variable) RETURN p,p1,p2", + query_workflows=f"MATCH p=(:GH_Repository {{node_id:'{rid}'}})-[:GH_Contains]->(:GH_Workflow)-[:GH_Contains]->(:GH_WorkflowJob)-[:GH_Contains]->(step:GH_WorkflowStep) OPTIONAL MATCH p1=(step)-[:GH_UsesSecret]->(:GH_Secret) OPTIONAL MATCH p2=(step)-[:GH_UsesVariable]->(:GH_Variable) RETURN p,p1,p2", query_runners=f"MATCH p=(:GH_Repository {{node_id:'{rid}'}})-[:GH_CanUseRunner]->(:GH_Runner) RETURN p", query_environments=f"MATCH p=(:GH_Repository {{node_id: '{rid}'}})-[:GH_HasEnvironment]->(:GH_Environment) RETURN p", query_secrets=f"MATCH p=(:GH_Repository {{node_id:'{rid}'}})-[:GH_HasSecret]->(:GH_Secret) RETURN p", diff --git a/src/openhound_github/models/saml_assertion_consumer_service.py b/src/openhound_github/models/saml_assertion_consumer_service.py new file mode 100644 index 0000000..b1d29d3 --- /dev/null +++ b/src/openhound_github/models/saml_assertion_consumer_service.py @@ -0,0 +1,102 @@ +from dataclasses import dataclass + +from openhound.core.asset import BaseAsset, EdgeDef, NodeDef +from openhound.core.models.entries_dataclass import Edge, EdgePath, EdgeProperties +from openhound_github.graph import GHNode, GHNodeProperties +from openhound_github.kinds import edges as ek +from openhound_github.kinds import nodes as nk +from openhound_github.main import app + +from .saml_helpers import ( + build_saml_route, + build_service_provider_node_id, + normalize_scope_type, +) + +@dataclass +class SAMLAssertionConsumerServiceProperties(GHNodeProperties): + """Properties for a normalized GitHub ACS route. + + Attributes: + native_id: The GitHub enterprise or organization node ID. + scope_type: The GitHub SAML scope type. + scope_slug: The GitHub enterprise or organization slug. + acs_url: The byte-exact GitHub ACS URL. + sp_entity_id: The byte-exact GitHub service provider entity ID. + """ + native_id: str | None = None + scope_type: str | None = None + scope_slug: str | None = None + acs_url: str | None = None + sp_entity_id: str | None = None + +@app.asset( + node=NodeDef( + kind=nk.SAML_ASSERTION_CONSUMER_SERVICE, + description="Normalized SAML assertion consumer service derived from GitHub SAML configuration", + icon="id-badge", + properties=SAMLAssertionConsumerServiceProperties, + ), + edges=[ + EdgeDef( + start=nk.SAML_SERVICE_PROVIDER, + end=nk.SAML_ASSERTION_CONSUMER_SERVICE, + kind=ek.SAML_HAS_ASSERTION_CONSUMER_SERVICE, + description="Normalized SAML service provider has this assertion consumer service", + traversable=False, + ), + ], +) +class SamlAssertionConsumerService(BaseAsset): + environment_slug: str + environment_type: str + environment_node_id: str | None = None + environment_name: str | None = None + + @property + def service_provider_node_id(self) -> str: + return build_service_provider_node_id( + self.environment_type, + self.environment_slug, + ) + + @property + def saml_route(self) -> tuple[str, str]: + return build_saml_route( + self.environment_type, + self.environment_slug, + ) + + @property + def node_id(self) -> str: + acs_url, _ = self.saml_route + return f"saml:acs:{acs_url}" + + @property + def as_node(self) -> GHNode: + acs_url, sp_entity_id = self.saml_route + scope_type = normalize_scope_type(self.environment_type) + + return GHNode( + kinds=[nk.SAML_ASSERTION_CONSUMER_SERVICE], + properties=SAMLAssertionConsumerServiceProperties( + name=self.node_id, + displayname=acs_url, + node_id=self.node_id, + acs_url=acs_url, + sp_entity_id=sp_entity_id, + environmentid=self.environment_node_id, + native_id=self.environment_node_id, + scope_type=scope_type, + scope_slug=self.environment_slug, + ), + ) + + @property + def edges(self): + yield Edge( + kind=ek.SAML_HAS_ASSERTION_CONSUMER_SERVICE, + start=EdgePath(value=self.service_provider_node_id, match_by="id"), + end=EdgePath(value=self.node_id, match_by="id"), + properties=EdgeProperties(traversable=False), + ) \ No newline at end of file diff --git a/src/openhound_github/models/saml_helpers.py b/src/openhound_github/models/saml_helpers.py new file mode 100644 index 0000000..96a9b31 --- /dev/null +++ b/src/openhound_github/models/saml_helpers.py @@ -0,0 +1,55 @@ +from urllib.parse import urlparse + +_FOREIGN_USER_KIND = { + "entra": "AZUser", + "okta": "Okta_User", + "pingone": "PingOne_User", +} + +def detect_foreign_idp( + issuer: str | None, sso_url: str | None +) -> tuple[str | None, str | None]: + """Return the canonical foreign IdP type and tenant/environment identifier.""" + if not issuer: + return None, None + + if issuer.startswith("https://auth.pingone.com/"): + return "pingone", issuer.split("/")[3] + + if issuer.startswith("https://sts.windows.net/"): + return "entra", issuer.split("/")[3] + + if issuer.startswith("http://www.okta.com/"): + domain = urlparse(sso_url).netloc if sso_url else None + return "okta", domain + + return None, None + +def foreign_user_kind(foreign_idp_type: str | None) -> str: + return _FOREIGN_USER_KIND.get(foreign_idp_type or "", "") + +def build_service_provider_node_id( + environment_type: str | None, + environment_slug: str | None +) -> str | None: + if not environment_type or not environment_slug: + return None + return f"saml:sp:github:{environment_type}:{environment_slug}" + +def build_issuer_node_id(issuer: str | None) -> str | None: + if not issuer: + return None + return f"saml:trusted-issuer:{issuer}" + +def build_saml_route( + environment_type: str, + environment_slug: str, +) -> tuple[str, str]: + if environment_type == "enterprise": + base = f"https://github.com/enterprises/{environment_slug}" + else: + base = f"https://github.com/orgs/{environment_slug}" + return f"{base}/saml/consume", base + +def normalize_scope_type(environment_type: str) -> str: + return "organization" if environment_type == "org" else environment_type \ No newline at end of file diff --git a/src/openhound_github/models/saml_issuer.py b/src/openhound_github/models/saml_issuer.py new file mode 100644 index 0000000..5a46370 --- /dev/null +++ b/src/openhound_github/models/saml_issuer.py @@ -0,0 +1,92 @@ +from dataclasses import dataclass + +from openhound.core.asset import BaseAsset, NodeDef, EdgeDef +from openhound.core.models.entries_dataclass import Edge, EdgePath, EdgeProperties +from openhound_github.graph import GHNode, GHNodeProperties +from openhound_github.kinds import nodes as nk +from openhound_github.kinds import edges as ek +from openhound_github.main import app + +from .saml_helpers import ( + build_issuer_node_id, + build_service_provider_node_id, + normalize_scope_type, +) + +@dataclass +class SAMLIssuerProperties(GHNodeProperties): + """Properties for a normalized GitHub trusted SAML issuer. + + Attributes: + native_id: The GitHub enterprise or organization node ID. + scope_type: The GitHub SAML scope type. + scope_slug: The GitHub enterprise or organization slug. + entity_id: The trusted SAML issuer entity ID. + """ + native_id: str | None = None + scope_type: str | None = None + scope_slug: str | None = None + entity_id: str | None = None + +@app.asset( + node=NodeDef( + kind=nk.SAML_ISSUER, + description="Normalized SAML issuer derived from GitHub SAML configuration", + icon="id-badge", + properties=SAMLIssuerProperties, + ), + edges=[ + EdgeDef( + start=nk.SAML_SERVICE_PROVIDER, + end=nk.SAML_ISSUER, + kind=ek.SAML_TRUSTS_ISSUER, + description="Normalized SAML service provider trusts this issuer", + traversable=False, + ), + ], +) +class SamlIssuer(BaseAsset): + issuer: str + environment_slug: str + environment_type: str + environment_node_id: str | None = None + environment_name: str | None = None + + @property + def node_id(self) -> str | None: + return build_issuer_node_id(self.issuer) + + @property + def service_provider_node_id(self) -> str: + return build_service_provider_node_id( + self.environment_type, + self.environment_slug, + ) + + @property + def as_node(self) -> GHNode: + scope_type = normalize_scope_type(self.environment_type) + + return GHNode( + kinds=[nk.SAML_ISSUER], + properties=SAMLIssuerProperties( + name=self.node_id, + displayname=self.issuer, + node_id=self.node_id, + entity_id=self.issuer, + environmentid=self.environment_node_id, + native_id=self.environment_node_id, + scope_type=scope_type, + scope_slug=self.environment_slug, + ), + ) + + @property + def edges(self): + if self.issuer: + yield Edge( + kind=ek.SAML_TRUSTS_ISSUER, + start=EdgePath(value=self.service_provider_node_id, match_by="id"), + end=EdgePath(value=self.node_id, match_by="id"), + properties=EdgeProperties(traversable=False), + ) \ No newline at end of file diff --git a/src/openhound_github/models/saml_provider.py b/src/openhound_github/models/saml_provider.py index 809af03..2d58ea8 100644 --- a/src/openhound_github/models/saml_provider.py +++ b/src/openhound_github/models/saml_provider.py @@ -8,6 +8,7 @@ from openhound_github.kinds import edges as ek from openhound_github.kinds import nodes as nk from openhound_github.main import app +from .saml_helpers import build_service_provider_node_id, detect_foreign_idp, foreign_user_kind @dataclass @@ -40,7 +41,7 @@ class GHSamlProviderProperties(GHNodeProperties): @app.asset( node=NodeDef( kind=nk.SAML_IDENTITY_PROVIDER, - description="GitHub Organization SAML Identity Provider", + description="SAML Identity Provider for GitHub Organization or Enterprise Account", icon="id-badge", properties=GHSamlProviderProperties, ), @@ -49,7 +50,14 @@ class GHSamlProviderProperties(GHNodeProperties): start=nk.ORGANIZATION, end=nk.SAML_IDENTITY_PROVIDER, kind=ek.HAS_SAML_IDENTITY_PROVIDER, - description="Org uses this SAML IdP", + description="GitHub Organization uses this SAML IdP", + traversable=False, + ), + EdgeDef( + start=nk.ENTERPRISE, + end=nk.SAML_IDENTITY_PROVIDER, + kind=ek.HAS_SAML_IDENTITY_PROVIDER, + description="GitHub Enterprise Account uses this SAML IdP", traversable=False, ), ], @@ -60,18 +68,19 @@ class SamlProvider(BaseAsset): id: str digest_method: str | None = None - idp_certificate: str | None = Field(default=None, alias="idpCertificate") + idp_certificate: str | None = None issuer: str | None = None - signature_method: str | None = Field(default=None, alias="signatureMethod") - sso_url: str | None = Field(default=None, alias="ssoUrl") + signature_method: str | None = None + sso_url: str | None = None # Detected foreign IdP type and tenant, derived from issuer/sso_url # foreign_idp_type: str | None = None # e.g. "entra", "okta", "pingone" foreign_environment_id: str | None = None # tenant/org ID in the foreign IdP # Additional - org_login: str - org_name: str - org_node_id: str # organization.id (GraphQL global ID) + environment_node_id: str # organization.id (GraphQL global ID) + environment_name: str + environment_slug: str + environment_type: str @property def node_id(self) -> str: @@ -80,23 +89,23 @@ def node_id(self) -> str: @property def as_node(self) -> GHNode: - iid = self.node_id + _, foreign_environment_id = detect_foreign_idp(self.issuer, self.sso_url) return GHNode( kinds=[nk.SAML_IDENTITY_PROVIDER], properties=GHSamlProviderProperties( - name=self.org_name, - displayname=self.org_name, - node_id=iid, + name=self.node_id, + displayname=self.environment_name, + node_id=self.node_id, issuer=self.issuer, sso_url=self.sso_url, signature_method=self.signature_method, digest_method=self.digest_method, idp_certificate=self.idp_certificate, - environment_name=self.org_name, - environmentid=self.org_node_id, - foreign_environment_id=self.foreign_environment_id, - query_environments=f"MATCH p=(:GH_SamlIdentityProvider {{node_id:'{iid}'}})<-[:GH_HasSamlIdentityProvider]-(:GH_Organization) RETURN p", - query_external_identities=f"MATCH p=(:GH_SamlIdentityProvider {{node_id:'{iid}'}})-[:GH_HasExternalIdentity]->() RETURN p", + environment_name=self.environment_name, + environmentid=self.environment_node_id, + foreign_environment_id=foreign_environment_id, + query_environments=f"MATCH p=(:GitHub)-[:GH_HasSamlIdentityProvider]->(:GH_SamlIdentityProvider {{node_id:'{self.node_id}'}}) RETURN p", + query_external_identities=f"MATCH p=(:GH_SamlIdentityProvider {{node_id:'{self.node_id}'}})-[:GH_HasExternalIdentity]->(:GH_ExternalIdentity) RETURN p", ), ) @@ -104,7 +113,7 @@ def as_node(self) -> GHNode: def edges(self): yield Edge( kind=ek.HAS_SAML_IDENTITY_PROVIDER, - start=EdgePath(value=self.org_node_id, match_by="id"), + start=EdgePath(value=self.environment_node_id, match_by="id"), end=EdgePath(value=self.node_id, match_by="id"), properties=EdgeProperties(traversable=False), ) diff --git a/src/openhound_github/models/saml_service_provider.py b/src/openhound_github/models/saml_service_provider.py new file mode 100644 index 0000000..7e045ba --- /dev/null +++ b/src/openhound_github/models/saml_service_provider.py @@ -0,0 +1,132 @@ +from dataclasses import dataclass + +from openhound.core.asset import BaseAsset, EdgeDef, NodeDef +from openhound.core.models.entries_dataclass import Edge, EdgePath, EdgeProperties +from pydantic import BaseModel + +from openhound_github.graph import GHNode, GHNodeProperties +from openhound_github.kinds import edges as ek +from openhound_github.kinds import nodes as nk +from openhound_github.main import app +from .saml_helpers import ( + build_issuer_node_id, + build_saml_route, + build_service_provider_node_id, + normalize_scope_type, +) + +@dataclass +class SAMLServiceProviderProperties(GHNodeProperties): + """Properties for a normalized GitHub SAML service provider. + + Attributes: + native_id: The GitHub enterprise or organization node ID. + scope_type: The GitHub SAML scope type. + scope_slug: The GitHub enterprise or organization slug. + saml_provider_id: The native GitHub SAML provider node ID. + enabled: Whether SAML is enabled for the scope. + entity_id: The SAML service provider entity ID. + environment_name: The name of the GitHub organization or enterprise. + """ + + native_id: str | None = None + scope_type: str | None = None + scope_slug: str | None = None + saml_provider_id: str | None = None + enabled: bool | None = None + entity_id: str | None = None + environment_name: str | None = None + +@app.asset( + node=NodeDef( + kind=nk.SAML_SERVICE_PROVIDER, + description="Normalized SAML service provider derived from GitHub SAML configuration", + icon="id-badge", + properties=SAMLServiceProviderProperties, + ), + edges=[ + EdgeDef( + start=nk.SAML_IDENTITY_PROVIDER, + end=nk.SAML_SERVICE_PROVIDER, + kind=ek.SAML_IMPLEMENTS, + description="GitHub native SAML configuration implements a normalized SAML service provider", + traversable=False, + ), + EdgeDef( + start=nk.ORGANIZATION, + end=nk.SAML_SERVICE_PROVIDER, + kind=ek.SAML_IMPLEMENTS, + description="GitHub Organization implements a normalized SAML service provider", + traversable=False, + ), + EdgeDef( + start=nk.ENTERPRISE, + end=nk.SAML_SERVICE_PROVIDER, + kind=ek.SAML_IMPLEMENTS, + description="GitHub Enterprise Account implements a SAML service provider", + traversable=False, + ), + ], +) + +class SamlServiceProvider(BaseAsset): + id: str + issuer: str | None = None + environment_node_id: str + environment_name: str + environment_slug: str + environment_type: str + + @property + def service_provider_node_id(self) -> str: + return build_service_provider_node_id( + self.environment_type, + self.environment_slug, + ) + + @property + def issuer_node_id(self) -> str | None: + return build_issuer_node_id(self.issuer) + + @property + def saml_route(self) -> tuple[str, str]: + return build_saml_route( + self.environment_type, + self.environment_slug, + ) + + @property + def as_node(self) -> GHNode: + _, sp_entity_id = self.saml_route + scope_type = normalize_scope_type(self.environment_type) + return GHNode( + kinds=[nk.SAML_SERVICE_PROVIDER], + properties=SAMLServiceProviderProperties( + name=self.service_provider_node_id, + displayname=self.environment_name, + node_id=self.service_provider_node_id, + entity_id=sp_entity_id, + environment_name=self.environment_name, + environmentid=self.environment_node_id, + native_id=self.environment_node_id, + scope_type=scope_type, + scope_slug=self.environment_slug, + saml_provider_id=self.id, + enabled=True, + ), + ) + + @property + def edges(self): + yield Edge( + kind=ek.SAML_IMPLEMENTS, + start=EdgePath(value=self.environment_node_id, match_by="id"), + end=EdgePath(value=self.service_provider_node_id, match_by="id"), + properties=EdgeProperties(traversable=False), + ) + yield Edge( + kind=ek.SAML_IMPLEMENTS, + start=EdgePath(value=self.id, match_by="id"), + end=EdgePath(value=self.service_provider_node_id, match_by="id"), + properties=EdgeProperties(traversable=False), + ) diff --git a/src/openhound_github/models/secret_scanning_alert.py b/src/openhound_github/models/secret_scanning_alert.py index 2b887d8..4b670d6 100644 --- a/src/openhound_github/models/secret_scanning_alert.py +++ b/src/openhound_github/models/secret_scanning_alert.py @@ -149,11 +149,13 @@ def org_node_id(self) -> str | None: @property def as_node(self) -> GHNode: aid = self.node_id + repo_name = self.repository.name if self.repository else self.org_login + label = self.secret_type or "secret alert" return GHNode( kinds=[nk.SECRET_SCANNING_ALERT], properties=GHSecretScanningAlertProperties( - name=str(self.number), - displayname=self.secret_type_display_name or str(self.number), + name=f"{repo_name}-{self.number}-{label}", + displayname=f"{label} in {repo_name}", node_id=aid, environmentid=self.org_node_id, repository_name=self.repository.name if self.repository else "", diff --git a/src/openhound_github/models/workflow.py b/src/openhound_github/models/workflow.py index 403301d..b15dbad 100644 --- a/src/openhound_github/models/workflow.py +++ b/src/openhound_github/models/workflow.py @@ -246,6 +246,9 @@ class GHWorkflowProperties(GHNodeProperties): branch: The branch where the workflow file was found. contents: The content of the workflow file. query_repository: Query for repository. + query_jobs: Query for workflow jobs. + query_execution: Query for workflow executions. + query_references: Query for workflow references (secrets and variables). query_editors: Query for editors. environment_name: The name of the environment (GitHub organization). """ @@ -263,6 +266,9 @@ class GHWorkflowProperties(GHNodeProperties): trigger_dispatch_inputs: list[str] | None = None is_pwn_requestable: bool = False query_repository: str | None = None + query_jobs: str | None = None + query_execution: str | None = None + query_references: str | None = None query_editors: str | None = None environment_name: str | None = None @@ -278,7 +284,7 @@ class GHWorkflowProperties(GHNodeProperties): EdgeDef( start=nk.REPOSITORY, end=nk.WORKFLOW, - kind=ek.HAS_WORKFLOW, + kind=ek.CONTAINS, description="Repository contains workflow", traversable=False, ), @@ -611,10 +617,13 @@ def as_node(self) -> GHNode: repository_id=self.repository_node_id, environment_name=self.org_login, environmentid=self.org_node_id, - query_repository=f"MATCH p=(:GH_Repository)-[:GH_HasWorkflow]->(:GH_Workflow {{node_id:'{wid}'}}) RETURN p", + query_repository=f"MATCH p=(:GH_Repository)-[:GH_Contains]->(:GH_Workflow {{node_id:'{wid}'}}) RETURN p", + query_jobs=f"MATCH p=(:GH_Workflow {{node_id:'{wid}'}})-[:GH_Contains]->(:GH_WorkflowJob) RETURN p", + query_execution=f"MATCH p=(:GH_Workflow {{node_id:'{wid}'}})-[:GH_Contains]->(:GH_WorkflowJob)-[:GH_Contains]->(:GH_WorkflowStep) RETURN p", + query_references=f"MATCH p=(:GH_Workflow {{node_id:'{wid}'}})-[:GH_Contains]->(:GH_WorkflowJob)-[:GH_Contains]->(step:GH_WorkflowStep) OPTIONAL MATCH p1=(step)-[:GH_UsesSecret]->() OPTIONAL MATCH p2=(step)-[:GH_UsesVariable]->() RETURN p,p1,p2", query_editors=( f"MATCH p=(role:GH_Role)-[:GH_HasRole|GH_HasBaseRole|GH_MemberOf|GH_WriteRepoContents|GH_WriteRepoPullRequests*1..]->" - f"(:GH_Repository)-[:GH_HasWorkflow]->(:GH_Workflow {{node_id:'{wid}'}}) " + f"(:GH_Repository)-[:GH_Contains]->(:GH_Workflow {{node_id:'{wid}'}}) " f"MATCH p1=(role)<-[:GH_HasRole]-(:GH_User) RETURN p,p1" ), ), @@ -623,7 +632,7 @@ def as_node(self) -> GHNode: @property def _has_workflow_edge(self): yield Edge( - kind=ek.HAS_WORKFLOW, + kind=ek.CONTAINS, start=EdgePath(value=self.repository_node_id, match_by="id"), end=EdgePath(value=self.node_id, match_by="id"), properties=EdgeProperties(traversable=False), diff --git a/src/openhound_github/models/workflow_job.py b/src/openhound_github/models/workflow_job.py index 9dad1f4..c986074 100644 --- a/src/openhound_github/models/workflow_job.py +++ b/src/openhound_github/models/workflow_job.py @@ -47,6 +47,9 @@ class GHWorkflowJobProperties(GHNodeProperties): repository_name: The containing repository name. repository_id: The containing repository node ID. environment_name: The name of the GitHub organization. + query_repository: Query for repository. + query_steps: Query for workflow steps. + query_references: Query for workflow references (secrets and variables). """ job_key: str | None = None @@ -60,6 +63,9 @@ class GHWorkflowJobProperties(GHNodeProperties): repository_name: str | None = None repository_id: str | None = None environment_name: str | None = None + query_repository: str | None = None + query_steps: str | None = None + query_references: str | None = None @app.asset( @@ -73,7 +79,7 @@ class GHWorkflowJobProperties(GHNodeProperties): EdgeDef( start=nk.WORKFLOW, end=nk.WORKFLOW_JOB, - kind=ek.HAS_JOB, + kind=ek.CONTAINS, description="Workflow contains job", traversable=False, ), @@ -220,6 +226,7 @@ def is_self_hosted(self) -> bool: @property def as_node(self) -> GHNode: + jid = self.node_id return GHNode( kinds=[nk.WORKFLOW_JOB], properties=GHWorkflowJobProperties( @@ -238,6 +245,9 @@ def as_node(self) -> GHNode: repository_id=self.repository_node_id, environment_name=self.org_login, environmentid=self.org_node_id, + query_repository=f"MATCH p=(repo:GH_Repository)-[:GH_Contains]->(:GH_Workflow)-[:GH_Contains]->(:GH_WorkflowJob {{node_id:'{jid}'}}) RETURN p", + query_steps=f"MATCH p=(:GH_WorkflowJob {{node_id:'{jid}'}})-[:GH_Contains]->(:GH_WorkflowStep) RETURN p", + query_references=f"MATCH p=(:GH_WorkflowJob {{node_id:'{jid}'}})-[:GH_Contains]->(step:GH_WorkflowStep) OPTIONAL MATCH p1=(step)-[:GH_UsesSecret]->() OPTIONAL MATCH p2=(step)-[:GH_UsesVariable]->() RETURN p,p1,p2", ), ) @@ -361,7 +371,7 @@ def _uses_variable_edges(self): @property def _has_job_edge(self): yield Edge( - kind=ek.HAS_JOB, + kind=ek.CONTAINS, start=EdgePath(value=self.workflow_node_id, match_by="id"), end=EdgePath(value=self.node_id, match_by="id"), properties=EdgeProperties(traversable=False), diff --git a/src/openhound_github/models/workflow_step.py b/src/openhound_github/models/workflow_step.py index 345249d..391d956 100644 --- a/src/openhound_github/models/workflow_step.py +++ b/src/openhound_github/models/workflow_step.py @@ -49,6 +49,8 @@ class GHWorkflowStepProperties(GHNodeProperties): repository_name: The containing repository name. repository_id: The containing repository node ID. environment_name: The name of the GitHub organization. + query_repository: Query for containing repository. + query_references: Query for secrets and variables referenced by the step. """ step_index: int | None = None @@ -66,6 +68,8 @@ class GHWorkflowStepProperties(GHNodeProperties): repository_name: str | None = None repository_id: str | None = None environment_name: str | None = None + query_repository: str | None = None + query_references: str | None = None @app.asset( @@ -79,7 +83,7 @@ class GHWorkflowStepProperties(GHNodeProperties): EdgeDef( start=nk.WORKFLOW_JOB, end=nk.WORKFLOW_STEP, - kind=ek.HAS_STEP, + kind=ek.CONTAINS, description="Workflow job contains step", traversable=False, ), @@ -161,6 +165,7 @@ def org_node_id(self) -> str | None: @property def as_node(self) -> GHNode: name = self.name or f"step-{self.step_index}" + sid = self.node_id return GHNode( kinds=[nk.WORKFLOW_STEP], properties=GHWorkflowStepProperties( @@ -183,6 +188,8 @@ def as_node(self) -> GHNode: repository_id=self.repository_node_id, environment_name=self.org_login, environmentid=self.org_node_id, + query_repository=f"MATCH p=(repo:GH_Repository)-[:GH_Contains]->(:GH_Workflow)-[:GH_Contains]->(:GH_WorkflowJob)-[:GH_Contains]->(:GH_WorkflowStep {{node_id:'{sid}'}}) RETURN p", + query_references=f"MATCH p=(:GH_WorkflowStep {{node_id: '{sid}'}})-[:GH_UsesSecret|GH_UsesVariable]->() RETURN p", ), ) @@ -257,7 +264,7 @@ def _uses_variable_edges(self): @property def _has_step_edge(self): yield Edge( - kind=ek.HAS_STEP, + kind=ek.CONTAINS, start=EdgePath(value=self.job_node_id, match_by="id"), end=EdgePath(value=self.node_id, match_by="id"), properties=EdgeProperties(traversable=False), diff --git a/src/openhound_github/resources/enterprise.py b/src/openhound_github/resources/enterprise.py index a5d114e..bd44a43 100644 --- a/src/openhound_github/resources/enterprise.py +++ b/src/openhound_github/resources/enterprise.py @@ -6,6 +6,7 @@ from openhound_github.graphql import ( ENTERPRISE_ADMINS_QUERY, ENTERPRISE_MEMBERS_QUERY, + ENTERPRISE_SAML_PROVIDER_QUERY, ENTERPRISE_QUERY, ENTERPRISE_SAML_QUERY, ) @@ -15,18 +16,21 @@ BaseUser, Enterprise, EnterpriseAdmin, - EnterpriseExternalIdentity, EnterpriseManagedUser, EnterpriseOrganization, EnterpriseRole, EnterpriseRoleTeam, EnterpriseRoleUser, - EnterpriseSamlProvider, EnterpriseTeam, EnterpriseTeamMember, EnterpriseTeamOrganization, EnterpriseTeamRole, EnterpriseUser, + SamlProvider, + SamlServiceProvider, + SamlAssertionConsumerService, + SamlIssuer, + ExternalIdentity, ) logger = logging.getLogger(__name__) @@ -37,35 +41,23 @@ class SourceContext: """Shared context for GitHub API access.""" client: RESTClient + sso_client: RESTClient | None = None org_name: str | None = None enterprise_name: str | None = None @app.resource(name="enterprise", columns=Enterprise, parallelized=True) def enterprise(ctx: SourceContext): - paginator = GraphQLCursorPaginator( - page_info_path="data.enterprise.organizations.pageInfo", - cursor_variable="after", - cursor_field="endCursor", - has_next_field="hasNextPage", - ) data = { "query": ENTERPRISE_QUERY, "variables": {"slug": ctx.enterprise_name, "after": None}, } try: - for page_data in ctx.client.paginate( - "/graphql", - method="POST", - json=data, - paginator=paginator, - data_selector="data", - ): - page_enterprise = page_data[0].get("enterprise") - - if page_enterprise: - yield page_enterprise + response = ctx.client.post("/graphql", json=data).json() + page_enterprise = (response.get("data") or {}).get("enterprise") + if page_enterprise: + yield page_enterprise except Exception as e: logger.error( f"Error in resource 'enterprise' processing enterprise '{ctx.enterprise_name}': {e}", @@ -78,13 +70,33 @@ def enterprise(ctx: SourceContext): name="enterprise_organizations", columns=EnterpriseOrganization, parallelized=True ) def enterprise_organizations(enterprise_data: Enterprise, ctx: SourceContext): - orgs = (enterprise_data.organizations or {}).get("nodes", []) - for org in orgs: - yield { - **org, - "enterprise_node_id": enterprise_data.id, - "enterprise_slug": ctx.enterprise_name, - } + paginator = GraphQLCursorPaginator( + page_info_path="data.enterprise.organizations.pageInfo", + cursor_variable="after", + cursor_field="endCursor", + has_next_field="hasNextPage", + ) + data = { + "query": ENTERPRISE_QUERY, + "variables": {"slug": ctx.enterprise_name, "after": None}, + } + + for page_data in ctx.client.paginate( + "/graphql", + method="POST", + json=data, + paginator=paginator, + data_selector="data", + ): + for enterprise_object in page_data: + es_data = enterprise_object.get("enterprise", {}) + orgs = (es_data.get("organizations") or {}).get("nodes", []) + for org in orgs: + yield { + **org, + "enterprise_node_id": enterprise_data.id, + "enterprise_slug": ctx.enterprise_name, + } @app.transformer(name="enterprise_members", columns=BaseUser, parallelized=True) @@ -330,48 +342,111 @@ def enterprise_admins(enterprise_data: Enterprise, ctx: SourceContext): @app.transformer( - name="enterprise_saml_provider", columns=EnterpriseSamlProvider, parallelized=True + name="enterprise_saml_provider", + table_name="saml_provider", + columns=SamlProvider, + parallelized=True ) def enterprise_saml_provider(enterprise_data: Enterprise, ctx: SourceContext): - paginator = GraphQLCursorPaginator( - page_info_path="data.enterprise.ownerInfo.samlIdentityProvider.externalIdentities.pageInfo", - cursor_variable="after", - cursor_field="endCursor", - has_next_field="hasNextPage", - allow_missing_page_info=True, - ) + client = ctx.sso_client + if not client: + logger.info( + "Skipping enterprise_saml_provider for enterprise '%s': no SSO client configured", + ctx.enterprise_name, + ) + return + data = { - "query": ENTERPRISE_SAML_QUERY, - "variables": {"slug": ctx.enterprise_name, "count": 1, "after": None}, + "query": ENTERPRISE_SAML_PROVIDER_QUERY, + "variables": {"slug": ctx.enterprise_name}, } - for page_data in ctx.client.paginate( - "/graphql", - method="POST", - json=data, - paginator=paginator, - data_selector="data", - ): - for enterprise_object in page_data: - es_data = enterprise_object.get("enterprise", {}) - saml_provider = (es_data.get("ownerInfo") or {}).get("samlIdentityProvider") - if not saml_provider: - return - yield { - **{k: v for k, v in saml_provider.items() if k != "externalIdentities"}, - "enterprise_node_id": enterprise_data.id, - "enterprise_slug": ctx.enterprise_name, - } + response = client.post("/graphql", json=data).json() + enterprise_object = (response.get("data") or {}).get("enterprise", {}) + saml_provider = (enterprise_object.get("ownerInfo") or {}).get("samlIdentityProvider") + if not saml_provider: + logger.warning( + "No enterprise SAML provider returned for enterprise '%s'", + ctx.enterprise_name, + ) + return + + yield { + **saml_provider, + "environment_node_id": enterprise_data.id, + "environment_name": enterprise_data.name, + "environment_slug": enterprise_data.slug, + "environment_type": "enterprise", + } + +@app.transformer( + name="enterprise_saml_service_provider", + table_name="saml_service_provider", + columns=SamlServiceProvider, + parallelized=True +) +def enterprise_saml_service_provider(saml_provider: SamlProvider, ctx: SourceContext): + yield { + "id": saml_provider["id"], + "issuer": saml_provider.get("issuer"), + "environment_node_id": saml_provider["environment_node_id"], + "environment_name": saml_provider["environment_name"], + "environment_slug": saml_provider["environment_slug"], + "environment_type": saml_provider["environment_type"], + } + +@app.transformer( + name="enterprise_saml_assertion_consumer_service", + table_name="saml_assertion_consumer_service", + columns=SamlAssertionConsumerService, + parallelized=True, +) +def enterprise_saml_assertion_consumer_service( + saml_provider: SamlProvider, ctx: SourceContext +): + yield { + "environment_slug": saml_provider.get("environment_slug"), + "environment_type": saml_provider.get("environment_type"), + "environment_node_id": saml_provider.get("environment_node_id"), + "environment_name": saml_provider.get("environment_name"), + } + +@app.transformer( + name="enterprise_saml_issuer", + table_name="saml_issuer", + columns=SamlIssuer, + parallelized=True +) +def enterprise_saml_issuer(saml_provider: SamlProvider, ctx: SourceContext): + issuer = saml_provider.get("issuer") + if not issuer: + return + yield { + "issuer": issuer, + "environment_slug": saml_provider.get("environment_slug"), + "environment_type": saml_provider.get("environment_type"), + "environment_node_id": saml_provider.get("environment_node_id"), + "environment_name": saml_provider.get("environment_name"), + } @app.transformer( - name="enterprise_external_identities", - columns=EnterpriseExternalIdentity, + name="enterprise_external_identity", + table_name="external_identities", + columns=ExternalIdentity, parallelized=True, ) -def enterprise_external_identities( - saml_provider: EnterpriseSamlProvider, ctx: SourceContext +def enterprise_external_identity( + saml_provider: SamlProvider, ctx: SourceContext ): + client = ctx.sso_client + if not client: + logger.info( + "Skipping enterprise_external_identity for enterprise '%s': no SSO client configured", + ctx.enterprise_name, + ) + return + paginator = GraphQLCursorPaginator( page_info_path="data.enterprise.ownerInfo.samlIdentityProvider.externalIdentities.pageInfo", cursor_variable="after", @@ -384,7 +459,7 @@ def enterprise_external_identities( "variables": {"slug": ctx.enterprise_name, "count": 100, "after": None}, } - for page_data in ctx.client.paginate( + for page_data in client.paginate( "/graphql", method="POST", json=data, @@ -395,17 +470,17 @@ def enterprise_external_identities( es_data = enterprise_object.get("enterprise", {}) page_provider = (es_data.get("ownerInfo") or {}).get("samlIdentityProvider") if not page_provider: + logger.warning( + "No enterprise SAML provider returned while fetching external identities for enterprise '%s'", + ctx.enterprise_name, + ) return for identity in (page_provider.get("externalIdentities") or {}).get( "nodes" ) or []: yield { **identity, - "saml_provider_id": saml_provider.id, - "saml_provider_issuer": saml_provider.issuer, - "saml_provider_sso_url": saml_provider.sso_url, - "enterprise_node_id": saml_provider.enterprise_node_id, - "enterprise_slug": saml_provider.enterprise_slug, + "environment_slug": saml_provider.get("environment_slug"), } @@ -415,8 +490,7 @@ def enterprise_resources(ctx: SourceContext): members_resource = enterprise_members(ctx) teams_resource = enterprise_teams(ctx) roles_resource = enterprise_roles(ctx) - saml_resource = enterprise_saml_provider(ctx) - return ( + resources = [ enterprise_resource, enterprise_resource | organizations_resource, enterprise_resource | members_resource | enterprise_users(ctx), @@ -430,6 +504,18 @@ def enterprise_resources(ctx: SourceContext): enterprise_resource | roles_resource | enterprise_role_teams(ctx), # enterprise_resource | enterprise_admin_roles(ctx), enterprise_resource | enterprise_admins(ctx), - enterprise_resource | saml_resource, - enterprise_resource | saml_resource | enterprise_external_identities(ctx), - ) + ] + + if ctx.sso_client: + saml_resource = enterprise_saml_provider(ctx) + resources.extend( + [ + enterprise_resource | saml_resource, + enterprise_resource | saml_resource | enterprise_saml_service_provider(ctx), + enterprise_resource | saml_resource | enterprise_saml_assertion_consumer_service(ctx), + enterprise_resource | saml_resource | enterprise_saml_issuer(ctx), + enterprise_resource | saml_resource | enterprise_external_identity(ctx), + ] + ) + + return tuple(resources) diff --git a/src/openhound_github/resources/organization.py b/src/openhound_github/resources/organization.py index 47dfa88..98589bd 100644 --- a/src/openhound_github/resources/organization.py +++ b/src/openhound_github/resources/organization.py @@ -59,6 +59,9 @@ RepoVariable, RunnerGroup, SamlProvider, + SamlServiceProvider, + SamlAssertionConsumerService, + SamlIssuer, ScimResource, SecretScanningAlert, SelectedOrgSecret, @@ -1707,15 +1710,16 @@ def saml_provider(ctx: SourceContext): response_data = response.get("data", {}) org_data = response_data.get("organization", {}) if response_data and org_data: - idp = org_data.get("samlIdentityProvider") - if not idp: + saml_provider = org_data.get("samlIdentityProvider") + if not saml_provider: continue yield { - **idp, - "org_node_id": org_data["id"], - "org_name": org_data["name"], - "org_login": org_name, + **saml_provider, + "environment_node_id": org_data["id"], + "environment_name": org_data["name"], + "environment_slug": org_name, + "environment_type": "org", } except Exception as e: logger.error( @@ -1730,7 +1734,6 @@ def external_identities(ctx: SourceContext): """Fetch external identities linked to the SAML provider. Args: - saml (SamlProvider): The SAML provider to extract identities for. ctx (SourceContext): The shared context containing the REST client and organization name. Yields: @@ -1767,7 +1770,10 @@ def external_identities(ctx: SourceContext): for identity in (idp.get("externalIdentities") or {}).get( "nodes" ) or []: - yield {**identity, "org_login": org_name} + yield { + **identity, + "environment_slug": org_data.get("login"), + } except Exception as e: logger.error( f"Error in resource 'external_identities' processing organization '{org_name}': {e}", @@ -1775,6 +1781,48 @@ def external_identities(ctx: SourceContext): ) continue +@app.transformer(name="saml_service_provider", columns=SamlServiceProvider, parallelized=True) +def saml_service_provider(saml_provider: SamlProvider, ctx: SourceContext): + """Transform SAML provider data into a normalized SAML interface representation. + + Args: + saml_provider (SamlProvider): The SAML provider to transform. + ctx (SourceContext): The shared context containing the REST client and organization name. + + Yields: + SamlInterface (SamlInterface): Normalized SAML interface record. + """ + yield { + "id": saml_provider["id"], + "issuer": saml_provider.get("issuer"), + "environment_node_id": saml_provider["environment_node_id"], + "environment_name": saml_provider["environment_name"], + "environment_slug": saml_provider["environment_slug"], + "environment_type": saml_provider["environment_type"], + } + +@app.transformer(name="saml_assertion_consumer_service", columns=SamlAssertionConsumerService, parallelized=True) +def saml_assertion_consumer_service(saml_provider: SamlProvider, ctx: SourceContext): + yield { + "environment_slug": saml_provider.get("environment_slug"), + "environment_type": saml_provider.get("environment_type"), + "environment_node_id": saml_provider.get("environment_node_id"), + "environment_name": saml_provider.get("environment_name"), + } + +@app.transformer(name="saml_issuer", columns=SamlIssuer, parallelized=True) +def saml_issuer(saml_provider: SamlProvider, ctx: SourceContext): + issuer = saml_provider.get("issuer") + if not issuer: + return + + yield { + "issuer": issuer, + "environment_slug": saml_provider.get("environment_slug"), + "environment_type": saml_provider.get("environment_type"), + "environment_node_id": saml_provider.get("environment_node_id"), + "environment_name": saml_provider.get("environment_name"), + } @app.resource(name="scim_users", columns=ScimResource, parallelized=True) def scim_users(ctx: SourceContext): @@ -1834,6 +1882,7 @@ def organization_resources(ctx: SourceContext): organization_secrets_resource = organization_secrets(ctx) organization_vars_resource = organization_variables(ctx) projected_enterprise_teams_resource = projected_enterprise_teams(ctx) + saml_resource = saml_provider(ctx) return ( org_resource, @@ -1860,7 +1909,10 @@ def organization_resources(ctx: SourceContext): repositories_graphql_resource | branches(ctx), branch_prot_rules_resource, secret_scanning_alerts(ctx), - saml_provider(ctx), + saml_resource, + saml_resource | saml_service_provider(ctx), + saml_resource | saml_assertion_consumer_service(ctx), + saml_resource | saml_issuer(ctx), external_identities(ctx), workflows_resource, workflows_resource | workflow_jobs(), diff --git a/src/openhound_github/source.py b/src/openhound_github/source.py index 5da35cb..873d1c8 100644 --- a/src/openhound_github/source.py +++ b/src/openhound_github/source.py @@ -8,6 +8,7 @@ from dlt.common.configuration import configspec from dlt.common.configuration.specs import CredentialsConfiguration from dlt.sources.helpers import requests +from dlt.sources.helpers.rest_client.auth import AuthConfigBase from dlt.sources.helpers.rest_client.auth import BearerTokenAuth from dlt.sources.helpers.rest_client.client import RESTClient from dlt.sources.helpers.rest_client.paginators import ( @@ -39,6 +40,7 @@ class OrgContext: class SourceContext: organizations: list[OrgContext] | None = field(default_factory=list) client: RESTClient | None = None + sso_client: RESTClient | None = None enterprise_name: str | None = None cache_lock: Lock = field(default_factory=Lock) app_cache: dict[str, dict[str, Any]] = field(default_factory=dict) @@ -66,6 +68,7 @@ class GithubEnterpriseAppCredentials(CredentialsConfiguration): app_id: str = None key_path: str = None enterprise_name: str = None + pat_token: str | None = None api_uri: str = "https://api.github.com" @property @@ -113,7 +116,7 @@ def source( host (str): The base GitHub API URL used for API calls. """ - def client(auth: GitHubAppInstallationAuth) -> RESTClient: + def client(auth: AuthConfigBase) -> RESTClient: return RESTClient( base_url=host, headers={ @@ -130,6 +133,8 @@ def client(auth: GitHubAppInstallationAuth) -> RESTClient: if credentials.auth == "enterprise_app": ctx = SourceContext(enterprise_name=credentials.enterprise_name) + if credentials.pat_token: + ctx.sso_client = client(BearerTokenAuth(token=credentials.pat_token)) github_app_session = GithubApp( client_id=credentials.client_id, private_key_path=credentials.key_path, diff --git a/src/openhound_github/transforms.py b/src/openhound_github/transforms.py index 8edc15c..bcc5630 100644 --- a/src/openhound_github/transforms.py +++ b/src/openhound_github/transforms.py @@ -1,6 +1,94 @@ import duckdb +def ensure_optional_input_tables( + con: duckdb.DuckDBPyConnection, schema: str = "github" +) -> None: + """Create typed empty tables for zero-row branch-policy inputs. + + DLT omits resources that yield no rows. Enterprise GitHub App collection can + legitimately have no branches, branch-protection rules, or repository roles, + while the derived branch transforms still need stable input schemas. + """ + con.execute(f""" + CREATE TABLE IF NOT EXISTS {schema}.branches ( + id VARCHAR, + branch_protection_rule JSON, + repository_node_id VARCHAR + ); + CREATE TABLE IF NOT EXISTS {schema}.branch_protection_rules ( + id VARCHAR, + repository_node_id VARCHAR, + pattern VARCHAR, + requires_approving_reviews BOOLEAN, + lock_branch BOOLEAN, + restricts_pushes BOOLEAN, + is_admin_enforced BOOLEAN, + bypass_pull_request_allowances JSON, + push_allowances JSON, + blocks_creations BOOLEAN + ); + CREATE TABLE IF NOT EXISTS {schema}.repo_roles ( + id BIGINT, + repository_node_id VARCHAR, + permissions JSON + ); + CREATE TABLE IF NOT EXISTS {schema}.organization_variables ( + name VARCHAR, + org_login VARCHAR, + value VARCHAR, + visibility VARCHAR, + selected_repositories_url VARCHAR, + created_at TIMESTAMP, + updated_at TIMESTAMP + ); + CREATE TABLE IF NOT EXISTS {schema}.selected_organization_variables ( + org_login VARCHAR, + variable_name VARCHAR, + repository_node_id VARCHAR + ); + """) + con.execute(f""" + ALTER TABLE {schema}.branches + ADD COLUMN IF NOT EXISTS branch_protection_rule JSON; + + ALTER TABLE {schema}.branch_protection_rules + ADD COLUMN IF NOT EXISTS id VARCHAR; + ALTER TABLE {schema}.branch_protection_rules + ADD COLUMN IF NOT EXISTS repository_node_id VARCHAR; + ALTER TABLE {schema}.branch_protection_rules + ADD COLUMN IF NOT EXISTS pattern VARCHAR; + ALTER TABLE {schema}.branch_protection_rules + ADD COLUMN IF NOT EXISTS requires_approving_reviews BOOLEAN; + ALTER TABLE {schema}.branch_protection_rules + ADD COLUMN IF NOT EXISTS lock_branch BOOLEAN; + ALTER TABLE {schema}.branch_protection_rules + ADD COLUMN IF NOT EXISTS restricts_pushes BOOLEAN; + ALTER TABLE {schema}.branch_protection_rules + ADD COLUMN IF NOT EXISTS is_admin_enforced BOOLEAN; + ALTER TABLE {schema}.branch_protection_rules + ADD COLUMN IF NOT EXISTS bypass_pull_request_allowances JSON; + ALTER TABLE {schema}.branch_protection_rules + ADD COLUMN IF NOT EXISTS push_allowances JSON; + ALTER TABLE {schema}.branch_protection_rules + ADD COLUMN IF NOT EXISTS blocks_creations BOOLEAN; + + ALTER TABLE {schema}.repo_roles + ADD COLUMN IF NOT EXISTS permissions JSON; + + ALTER TABLE {schema}.organization_variables + ADD COLUMN IF NOT EXISTS name VARCHAR; + ALTER TABLE {schema}.organization_variables + ADD COLUMN IF NOT EXISTS org_login VARCHAR; + + ALTER TABLE {schema}.selected_organization_variables + ADD COLUMN IF NOT EXISTS org_login VARCHAR; + ALTER TABLE {schema}.selected_organization_variables + ADD COLUMN IF NOT EXISTS variable_name VARCHAR; + ALTER TABLE {schema}.selected_organization_variables + ADD COLUMN IF NOT EXISTS repository_node_id VARCHAR; + """) + # TODO: # This can be optimized to generate the actor_branch_gates table # in one go instead of intermedaite tables @@ -93,6 +181,8 @@ def transforms(con: duckdb.DuckDBPyConnection, schema: str = "github") -> None: con: The DuckDB connection to use for creating computed tables. schema: The DuckDB schema name containing the source tables. """ + + ensure_optional_input_tables(con, schema) join_branch_bpr(con, schema) actor_allowances(con, schema) unprotected_branches(con, schema) diff --git a/tests/test_enterprise_resources.py b/tests/test_enterprise_resources.py new file mode 100644 index 0000000..25cb818 --- /dev/null +++ b/tests/test_enterprise_resources.py @@ -0,0 +1,93 @@ +from types import SimpleNamespace + +from openhound_github.resources.enterprise import ( + SourceContext, + enterprise, + enterprise_organizations, +) + + +class _FakeResponse: + def __init__(self, payload: dict): + self._payload = payload + + def json(self) -> dict: + return self._payload + + +class _FakeClient: + def __init__(self, payload: dict, pages: list[dict] | None = None): + self.payload = payload + self.pages = pages or [] + self.post_calls: list[tuple[str, dict]] = [] + self.paginate_calls: list[tuple[str, dict]] = [] + + def post(self, path: str, json: dict): + self.post_calls.append((path, json)) + return _FakeResponse(self.payload) + + def paginate(self, path: str, **kwargs): + self.paginate_calls.append((path, kwargs)) + return iter(self.pages) + + +def test_enterprise_resource_yields_single_record() -> None: + client = _FakeClient( + { + "data": { + "enterprise": { + "id": "E_1", + "slug": "acme", + "organizations": { + "nodes": [{"id": "O_1", "login": "org-1"}], + "pageInfo": {"hasNextPage": True, "endCursor": "cursor-1"}, + }, + } + } + } + ) + ctx = SourceContext(client=client, enterprise_name="acme") + + rows = list(enterprise(ctx)) + + assert len(rows) == 1 + assert rows[0].id == "E_1" + assert len(client.post_calls) == 1 + + +def test_enterprise_organizations_paginates_all_pages() -> None: + client = _FakeClient( + payload={}, + pages=[ + [ + { + "enterprise": { + "organizations": { + "nodes": [{"id": "O_1", "login": "org-1"}], + "pageInfo": {"hasNextPage": True, "endCursor": "cursor-1"}, + } + } + } + ], + [ + { + "enterprise": { + "organizations": { + "nodes": [{"id": "O_2", "login": "org-2"}], + "pageInfo": {"hasNextPage": False, "endCursor": None}, + } + } + } + ], + ], + ) + ctx = SourceContext(client=client, enterprise_name="acme") + enterprise_data = SimpleNamespace(id="E_1") + + rows = list(enterprise_organizations.__wrapped__(enterprise_data, ctx)) + + assert rows == [ + {"id": "O_1", "login": "org-1", "enterprise_node_id": "E_1", "enterprise_slug": "acme"}, + {"id": "O_2", "login": "org-2", "enterprise_node_id": "E_1", "enterprise_slug": "acme"}, + ] + assert len(client.paginate_calls) == 1