added kubernetes crd parsing

docs/configs/kubernetes.md

@@ -143,3 +143,41 @@ If the `href` attribute is not present, Homepage will ignore the specific Ingres
 ## Caveats
 
 Similarly to Docker service discovery, there currently is no rigid ordering to discovered services and discovered services will be displayed above those specified in the `services.yaml`.
+
+## CRDs
+
+Homepage also comes with Kubernetes CRDs for services. These CRDs have same structure and properties as regular service YAML definition, with added properties of `group`, `weight`, `podSelector` and `instances`, used as described above. 
+
+Compared to annotations, CRD approach can use Kubernetes secrets and configMaps to populate attributes of the `widget` object. To do this, instead of using the name, ex. `key`, use `keyFrom`, to match kubernetes standards. Then use either `secretKeyRef` or `configMapKeyRef` object, see example:
+
+```yaml
+apiVersion: gethomepage.dev/v1
+kind: HomepageService
+metadata:
+  labels:
+    app.kubernetes.io/instance: sonarr
+  name: sonarr
+  namespace: media
+spec:
+  description: TV Show Management Application
+  group: Media
+  href: 'https://sonarr.example.org/'
+  icon: sonarr.svg
+  widget:
+    type: sonarr
+    url: 'http://sonarr.media.svc.cluster.local:8989'
+    keyFrom:
+      secretKeyRef:
+        key: SONARR_API_KEY
+        name: arr-secrets
+```
+
+Some attributes have values inferred from the definition itself, `app` is read from *app.kubernetes.io/name* annotation or metadata.name, namespace is also read from metadata.
+
+For secrets and configMaps, if `namespace` is not specified in ref object, it assumes it's in the same namespace as CRD. If you want to specify a default namespace (for example for security reason when assigning secrets:read permission for only one namespace), you can specify `defaultSecretNamespace` and `defaultConfigMapNamespace` in `kubernetes.yaml` file.
+
+```yaml
+---
+mode: cluster
+defaultSecretNamespace: homepage
+```

src/pages/api/kubernetes/stats/[...service].js

@@ -1,6 +1,6 @@
 import { CoreV1Api, Metrics } from "@kubernetes/client-node";
 
-import getKubeConfig from "../../../../utils/config/kubernetes";
+import getKubernetesConfig, { makeKubeConfig } from "../../../../utils/config/kubernetes";
 import { parseCpu, parseMemory } from "../../../../utils/kubernetes/kubernetes-utils";
 import createLogger from "../../../../utils/logger";
 
@@ -20,7 +20,7 @@ export default async function handler(req, res) {
   const labelSelector = podSelector !== undefined ? podSelector : `${APP_LABEL}=${appName}`;
 
   try {
-    const kc = getKubeConfig();
+    const kc = makeKubeConfig(getKubernetesConfig());
     if (!kc) {
       res.status(500).send({
         error: "No kubernetes configuration",

src/pages/api/kubernetes/status/[...service].js

@@ -1,6 +1,6 @@
 import { CoreV1Api } from "@kubernetes/client-node";
 
-import getKubeConfig from "../../../../utils/config/kubernetes";
+import getKubernetesConfig, { makeKubeConfig } from "../../../../utils/config/kubernetes";
 import createLogger from "../../../../utils/logger";
 
 const logger = createLogger("kubernetesStatusService");
@@ -18,7 +18,7 @@ export default async function handler(req, res) {
   }
   const labelSelector = podSelector !== undefined ? podSelector : `${APP_LABEL}=${appName}`;
   try {
-    const kc = getKubeConfig();
+    const kc = makeKubeConfig(getKubernetesConfig())
     if (!kc) {
       res.status(500).send({
         error: "No kubernetes configuration",

src/pages/api/widgets/kubernetes.js

@@ -1,6 +1,6 @@
 import { CoreV1Api, Metrics } from "@kubernetes/client-node";
 
-import getKubeConfig from "../../../utils/config/kubernetes";
+import getKubernetesConfig, { makeKubeConfig } from "../../../utils/config/kubernetes";
 import { parseCpu, parseMemory } from "../../../utils/kubernetes/kubernetes-utils";
 import createLogger from "../../../utils/logger";
 
@@ -8,7 +8,7 @@ const logger = createLogger("kubernetes-widget");
 
 export default async function handler(req, res) {
   try {
-    const kc = getKubeConfig();
+    const kc = makeKubeConfig(getKubernetesConfig())
     if (!kc) {
       return res.status(500).send({
         error: "No kubernetes configuration",

src/utils/config/kubernetes.js

@@ -6,13 +6,16 @@ import { KubeConfig } from "@kubernetes/client-node";
 
 import checkAndCopyConfig, { CONF_DIR, substituteEnvironmentVars } from "utils/config/config";
 
-export default function getKubeConfig() {
+export default function getKubernetesConfig() {
   checkAndCopyConfig("kubernetes.yaml");
 
   const configFile = path.join(CONF_DIR, "kubernetes.yaml");
   const rawConfigData = readFileSync(configFile, "utf8");
   const configData = substituteEnvironmentVars(rawConfigData);
-  const config = yaml.load(configData);
+  return yaml.load(configData);
+}
+
+export function makeKubeConfig(config) {
   const kc = new KubeConfig();
 
   switch (config?.mode) {

src/utils/config/service-helpers.js

@@ -3,12 +3,12 @@ import path from "path";
 
 import yaml from "js-yaml";
 import Docker from "dockerode";
-import { CustomObjectsApi, NetworkingV1Api, ApiextensionsV1Api } from "@kubernetes/client-node";
+import { CustomObjectsApi, NetworkingV1Api, ApiextensionsV1Api, CoreV1Api } from "@kubernetes/client-node";
 
 import createLogger from "utils/logger";
 import checkAndCopyConfig, { CONF_DIR, getSettings, substituteEnvironmentVars } from "utils/config/config";
 import getDockerArguments from "utils/config/docker";
-import getKubeConfig from "utils/config/kubernetes";
+import getKubernetesConfig, { makeKubeConfig } from "utils/config/kubernetes";
 import * as shvl from "utils/config/shvl";
 
 const logger = createLogger("service-helpers");
@@ -183,15 +183,16 @@ export async function servicesFromKubernetes() {
   const ANNOTATION_WIDGET_BASE = `${ANNOTATION_BASE}/widget.`;
   const { instanceName } = getSettings();
 
-  checkAndCopyConfig("kubernetes.yaml");
 
   try {
-    const kc = getKubeConfig();
+    const config = getKubernetesConfig();
+    const kc = makeKubeConfig(config);
     if (!kc) {
       return [];
     }
     const networking = kc.makeApiClient(NetworkingV1Api);
     const crd = kc.makeApiClient(CustomObjectsApi);
+    const core = kc.makeApiClient(CoreV1Api);
 
     const ingressList = await networking
       .listIngressForAllNamespaces(null, null, null, null)
@@ -203,6 +204,7 @@ export async function servicesFromKubernetes() {
 
     const traefikContainoExists = await checkCRD(kc, "ingressroutes.traefik.containo.us");
     const traefikExists = await checkCRD(kc, "ingressroutes.traefik.io");
+    const homepageServiceExists = await checkCRD(kc, "homepageservices.gethomepage.dev")
 
     const traefikIngressListContaino = await crd
       .listClusterCustomObject("traefik.containo.us", "v1alpha1", "ingressroutes")
@@ -245,65 +247,138 @@ export async function servicesFromKubernetes() {
       ingressList.items.push(...traefikServices);
     }
 
-    if (!ingressList) {
+    if (!ingressList && !homepageServiceExists) {
       return [];
     }
-    const services = ingressList.items
+
-      .filter(
+    let homepageServices = []
-        (ingress) =>
+    if (homepageServiceExists) {
-          ingress.metadata.annotations &&
+      homepageServices = await crd
-          ingress.metadata.annotations[`${ANNOTATION_BASE}/enabled`] === "true" &&
+        .listClusterCustomObject("gethomepage.dev", "v1", "homepageservices")
-          (!ingress.metadata.annotations[`${ANNOTATION_BASE}/instance`] ||
+        .then((response) => response.body)
-            ingress.metadata.annotations[`${ANNOTATION_BASE}/instance`] === instanceName ||
+        .catch(async (error) => {
-            `${ANNOTATION_BASE}/instance.${instanceName}` in ingress.metadata.annotations),
+          logger.error(
-      )
+            "Error getting services from gethomepage.dev crd: %d %s %s",
-      .map((ingress) => {
+            error.statusCode,
-        let constructedService = {
+            error.body,
-          app: ingress.metadata.annotations[`${ANNOTATION_BASE}/app`] || ingress.metadata.name,
+            error.response,
-          namespace: ingress.metadata.namespace,
+          );
-          href: ingress.metadata.annotations[`${ANNOTATION_BASE}/href`] || getUrlFromIngress(ingress),
+          return [];
-          name: ingress.metadata.annotations[`${ANNOTATION_BASE}/name`] || ingress.metadata.name,
-          group: ingress.metadata.annotations[`${ANNOTATION_BASE}/group`] || "Kubernetes",
-          weight: ingress.metadata.annotations[`${ANNOTATION_BASE}/weight`] || "0",
-          icon: ingress.metadata.annotations[`${ANNOTATION_BASE}/icon`] || "",
-          description: ingress.metadata.annotations[`${ANNOTATION_BASE}/description`] || "",
-          external: false,
-          type: "service",
-        };
-        if (ingress.metadata.annotations[`${ANNOTATION_BASE}/external`]) {
-          constructedService.external =
-            String(ingress.metadata.annotations[`${ANNOTATION_BASE}/external`]).toLowerCase() === "true";
-        }
-        if (ingress.metadata.annotations[`${ANNOTATION_BASE}/pod-selector`] !== undefined) {
-          constructedService.podSelector = ingress.metadata.annotations[`${ANNOTATION_BASE}/pod-selector`];
-        }
-        if (ingress.metadata.annotations[`${ANNOTATION_BASE}/ping`]) {
-          constructedService.ping = ingress.metadata.annotations[`${ANNOTATION_BASE}/ping`];
-        }
-        if (ingress.metadata.annotations[`${ANNOTATION_BASE}/siteMonitor`]) {
-          constructedService.siteMonitor = ingress.metadata.annotations[`${ANNOTATION_BASE}/siteMonitor`];
-        }
-        if (ingress.metadata.annotations[`${ANNOTATION_BASE}/statusStyle`]) {
-          constructedService.statusStyle = ingress.metadata.annotations[`${ANNOTATION_BASE}/statusStyle`];
-        }
-        Object.keys(ingress.metadata.annotations).forEach((annotation) => {
-          if (annotation.startsWith(ANNOTATION_WIDGET_BASE)) {
-            shvl.set(
-              constructedService,
-              annotation.replace(`${ANNOTATION_BASE}/`, ""),
-              ingress.metadata.annotations[annotation],
-            );
-          }
         });
-
+      await Promise.all(homepageServices.items.map(async (service) => {
-        try {
+        const { spec, metadata } = service;
-          constructedService = JSON.parse(substituteEnvironmentVars(JSON.stringify(constructedService)));
+        // Enter default values from metadata or annotations
-        } catch (e) {
+        spec.app = spec.app || metadata.annotations["app.kubernetes.io/name"] || metadata.name
-          logger.error("Error attempting k8s environment variable substitution.");
+        spec.namespace = spec.namespace || metadata.namespace
+        spec.name = spec.name || metadata.name
+        spec.weight = spec.weight || "0"
+        spec.icon = spec.icon || ""
+        spec.description = spec.description || ""
+        // Parse values from secrets and configmaps in widget
+        if (spec.widget) {
+          const parsedWidget = {}
+          await Promise.all(Object.keys(spec.widget).map(async key => {
+            // To keep up with kubernetes standard valueFrom
+            if (key.endsWith("From")) {
+              if (spec.widget[key].secretKeyRef) {
+                const { secretKeyRef } = spec.widget[key]
+                const secret = await core.readNamespacedSecret(secretKeyRef.name, secretKeyRef.namespace || config.defaultSecretNamespace || metadata.namespace)
+                const base64secret = secret.body.data[secretKeyRef.key]
+                if (!base64secret) {
+                  logger.error(
+                    "Error getting secret value: Secret %s in namespace %s doesn't contain key %s",
+                    spec.widget[key].secretKeyRef.name,
+                    metadata.namespace,
+                    spec.widget[key].secretKeyRef.key,
+                  );
+                  return
+                }
+                parsedWidget[key.substring(0, key.length - 4)] = Buffer.from(base64secret, "base64").toString("utf8")
+              } else if (spec.widget[key].configMapKeyRef) {
+                const { configMapKeyRef } = spec.widget[key]
+                const configMap = await core.readNamespacedConfigMap(configMapKeyRef.name, configMapKeyRef.namespace || config.defaultConfigMapNamespace || metadata.namespace)
+                const configMapValue = configMap.body.data[configMapKeyRef.key]
+                if (!configMapValue) {
+                  logger.error(
+                    "Error getting configMap value: ConfigMap %s in namespace %s doesn't contain key %s",
+                    spec.widget[key].configMapKey.name,
+                    metadata.namespace,
+                    spec.widget[key].configMapKey.key,
+                  );
+                  return
+                }
+                parsedWidget[key.substring(0, key.length - 4)] = configMapValue
+              }
+            } else {
+              parsedWidget[key] = spec.widget[key]
+            }
+          }))
+          spec.widget = parsedWidget
         }
+      }))
+    }
 
-        return constructedService;
+    const services = [
-      });
+      ...homepageServices.items
+        .filter(service => !service.spec.instances || service.spec.instances.includes(instanceName))
+        .map(service => service.spec),
+      ...ingressList.items
+        .filter(
+          (ingress) =>
+            ingress.metadata.annotations &&
+            ingress.metadata.annotations[`${ANNOTATION_BASE}/enabled`] === "true" &&
+            (!ingress.metadata.annotations[`${ANNOTATION_BASE}/instance`] ||
+              ingress.metadata.annotations[`${ANNOTATION_BASE}/instance`] === instanceName ||
+              `${ANNOTATION_BASE}/instance.${instanceName}` in ingress.metadata.annotations),
+        )
+        .map((ingress) => {
+          let constructedService = {
+            app: ingress.metadata.annotations[`${ANNOTATION_BASE}/app`] || ingress.metadata.name,
+            namespace: ingress.metadata.namespace,
+            href: ingress.metadata.annotations[`${ANNOTATION_BASE}/href`] || getUrlFromIngress(ingress),
+            name: ingress.metadata.annotations[`${ANNOTATION_BASE}/name`] || ingress.metadata.name,
+            group: ingress.metadata.annotations[`${ANNOTATION_BASE}/group`] || "Kubernetes",
+            weight: ingress.metadata.annotations[`${ANNOTATION_BASE}/weight`] || "0",
+            icon: ingress.metadata.annotations[`${ANNOTATION_BASE}/icon`] || "",
+            description: ingress.metadata.annotations[`${ANNOTATION_BASE}/description`] || "",
+            external: false,
+            type: "service",
+          };
+          if (ingress.metadata.annotations[`${ANNOTATION_BASE}/external`]) {
+            constructedService.external =
+              String(ingress.metadata.annotations[`${ANNOTATION_BASE}/external`]).toLowerCase() === "true";
+          }
+          if (ingress.metadata.annotations[`${ANNOTATION_BASE}/pod-selector`] !== undefined) {
+            constructedService.podSelector = ingress.metadata.annotations[`${ANNOTATION_BASE}/pod-selector`];
+          }
+          if (ingress.metadata.annotations[`${ANNOTATION_BASE}/ping`]) {
+            constructedService.ping = ingress.metadata.annotations[`${ANNOTATION_BASE}/ping`];
+          }
+          if (ingress.metadata.annotations[`${ANNOTATION_BASE}/siteMonitor`]) {
+            constructedService.siteMonitor = ingress.metadata.annotations[`${ANNOTATION_BASE}/siteMonitor`];
+          }
+          if (ingress.metadata.annotations[`${ANNOTATION_BASE}/statusStyle`]) {
+            constructedService.statusStyle = ingress.metadata.annotations[`${ANNOTATION_BASE}/statusStyle`];
+          }
+          Object.keys(ingress.metadata.annotations).forEach((annotation) => {
+            if (annotation.startsWith(ANNOTATION_WIDGET_BASE)) {
+              shvl.set(
+                constructedService,
+                annotation.replace(`${ANNOTATION_BASE}/`, ""),
+                ingress.metadata.annotations[annotation],
+              );
+            }
+          });
+
+          try {
+            constructedService = JSON.parse(substituteEnvironmentVars(JSON.stringify(constructedService)));
+          } catch (e) {
+            logger.error("Error attempting k8s environment variable substitution.");
+          }
+
+          return constructedService;
+        })
+    ];
 
     const mappedServiceGroups = [];
 
@@ -328,6 +403,7 @@ export async function servicesFromKubernetes() {
 
     return mappedServiceGroups;
   } catch (e) {
+    console.log(e)
     if (e) logger.error(e);
     throw e;
   }