Added UDP Output to be consumed by Logstash

README.md

@@ -99,6 +99,8 @@ github_access_tokens: # provide at least one token
   - 'token two'
 webhook: '' # URL to a POST webhook.
 webhook_payload: '' # Payload to POST to the webhook URL
+logstash: '' # IP Address for Logstash Instance (UDP)
+logstash_port: '' # Listen Packet Port
 blacklisted_extensions: [] # list of extensions to ignore
 blacklisted_paths: [] # list of paths to ignore
 blacklisted_entropy_extensions: [] # additional extensions to ignore for entropy checks

config.yaml

@@ -2,6 +2,9 @@ github_access_tokens:
   - ''
 webhook: '' # URL to which the payload is POSTed
 
+logstash: '' # IP Address for Logstash Instance (UDP)
+logstash_port: '' # Listen Packet Port
+
 # This default payload will work for Slack and MatterMost.
 # Consult your webhook API for additional configurations.
 webhook_payload: |

core/config.go

@@ -15,6 +15,8 @@ type Config struct {
 	GitHubAccessTokens           []string          `yaml:"github_access_tokens"`
 	Webhook                      string            `yaml:"webhook,omitempty"`
 	WebhookPayload               string            `yaml:"webhook_payload,omitempty"`
+	Logstash                     string            `yaml:"logstash,omitempty"`
+	LogstashPort	             string            `yaml:"logstash_port,omitempty"`
 	BlacklistedExtensions        []string          `yaml:"blacklisted_extensions"`
 	BlacklistedPaths             []string          `yaml:"blacklisted_paths"`
 	BlacklistedEntropyExtensions []string          `yaml:"blacklisted_entropy_extensions"`
@@ -72,6 +74,10 @@ func ParseConfig(options *Options) (*Config, error) {
 	if len(config.Webhook) > 0 {
 		config.Webhook = os.ExpandEnv(config.Webhook)
 	}
+
+	if len(config.Logstash) > 0 {
+		config.Logstash = os.ExpandEnv(config.Logstash)
+	}
 
 	return config, nil
 }

core/log.go

@@ -3,6 +3,7 @@ package core
 import (
 	"fmt"
 	"net/http"
+	"net"
 	"os"
 	"regexp"
 	"strings"
@@ -66,7 +67,24 @@ func (l *Logger) Log(level int, format string, args ...interface{}) {
 		payload := fmt.Sprintf(session.Config.WebhookPayload, text)
 		http.Post(session.Config.Webhook, "application/json", strings.NewReader(payload))
 	}
-
+
+	if session.Config.Logstash != "" {
+		text := colorStrip(fmt.Sprintf(format, args...))
+		payload := fmt.Sprintf(session.Config.Logstash, text)
+  		pc, err := net.ListenPacket("udp4", ":" + session.Config.LogstashPort)
+  		if err != nil {
+    		panic(err)
+  		}
+  		defer pc.Close()
+
+  		addr,err := net.ResolveUDPAddr("udp4", session.Config.Logstash)
+  		if err != nil {
+    		panic(err)
+  		}
+
+  		pc.WriteTo([]byte(payload), addr)	
+	}
+
 	if level == FATAL {
 		os.Exit(1)
 	}

go.mod

@@ -1,5 +1,7 @@
 module github.com/eth0izzle/shhgit
 
+go 1.14
+
 require (
 	github.com/fatih/color v1.7.0
 	github.com/google/go-github v17.0.0+incompatible

logstash/README.md

@@ -0,0 +1,29 @@
+# Logstash Configuration
+
+## 1. Install ElasticSearch, Kibana, and Logstash
+- Follow the instructions on Elastic's website: https://www.elastic.co/guide/en/elastic-stack-get-started/current/get-started-elastic-stack.html
+
+## 2. Configure Logstash
+
+In the folder logstash/config/01-shhgit-pipeline.conf is a working example with a few Grok rules to process the incoming messages from shhgit.  These Grok rules are imperfect, but should provide a starting point to improve upon.  
+
+- 01-shhgit-pipeline.conf should be placed in the Logstash /etc/logstash/conf.d directory.
+- Update the output elasticsearch to point to your elasticsearch cluster
+
+## 3. Configure Kibana
+
+### Configure Pattern
+- Click the gear icon (management) in the lower left
+- Click Kibana -> Index Patters
+- Click Create New Index Pattern
+- Type "shhgit-*" into the input box, then click Next Step
+
+### Import Dashboard
+In your web browser go to Kibana's IP using port 5601 (ex: 192.168.0.1:5601)
+- Click Management -> Saved Objects
+- You can import the dashboard found in the dashboard folder via the Import button in the top-right corner.
+  - shhgit Dashboard
+
+## TODO
+- Clean up Grok Rules
+- Add additional Grok Rules

logstash/config/01-shhgit-pipeline.conf

@@ -0,0 +1,100 @@
+input {
+  udp {
+    port => 8080
+  }
+}
+
+filter {
+
+  if "Matching file" in [message]  {
+    grok {
+      match => { "message" => "\[%{GREEDYDATA:location}\] Matching %{WORD:type} %{URIPATHPARAM:path} for %{GREEDYDATA:reason}\"" }
+    }
+  }
+
+  if "match for" in [message]  {
+    grok {
+      match => { "message" => "\[%{GREEDYDATA:location}\] %{NUMBER:number} match for %{GREEDYDATA:reason} in %{WORD:type} %{URIPATHPARAM:path}: %{URIPROTO:uriproto}://(?:%{USER:user}(?::%{GREEDYDATA:password}[^@]*)?@)?(?:%{URIHOST:urihost})?(?:%{URIPATHPARAM:uripathparam})?" }
+    }
+  }
+
+  if "Potential secret" in [message]  {
+    grok {
+      match => { "message" => "\[%{GREEDYDATA:location}\] %{GREEDYDATA:reason} in %{URIPATHPARAM:path} =%{SPACE}%{GREEDYDATA:secret}\"" }
+    }
+  }
+
+  if "matches for" in [message]  {
+    grok {
+      match => { "message" => "\[%{GREEDYDATA:location}\] %{NUMBER:number} matches for %{GREEDYDATA:reason} in %{WORD:type} %{GREEDYDATA:paths}\"" }
+    }
+  }
+
+  if "Google OAuth Key" in [message]  {
+    grok {
+      match => { "message" => "\[%{GREEDYDATA:location}\] %{NUMBER:number} matches for %{GREEDYDATA:reason} in %{WORD:type} %{GREEDYDATA:paths}\"" }
+    }
+  }
+
+  if "Google Cloud API Key" in [message]  {
+    grok {
+      match => { "message" => "\[%{GREEDYDATA:location}\] %{NUMBER:number} match for %{GREEDYDATA:reason} in %{WORD:type} %{URIPATHPARAM:path}:%{SPACE}%{GREEDYDATA:key}\"" }
+    }
+  }
+
+  mutate {
+    split => {"paths" => "," }
+  }
+
+  if "Cloning in to" in [message]  {
+    grok
+    {
+      match => { "message" => "\[%{GREEDYDATA:location}\] Cloning in to %{URIPATH:path}\"" }
+      add_tag => ["cloning"]
+    }
+  }
+
+  if "Failed to" in [message]  {
+    grok
+    {
+      match => { "message" => "\"%{GREEDYDATA:errorMessage}\"" }
+      add_tag => ["failed"]
+    }
+  }
+
+  if "Cloning failed" in [message]  {
+    grok
+    {
+      match => { "message" => "\"%{GREEDYDATA:errorMessage}\"" }
+      add_tag => ["failed"]
+    }
+  }
+
+  if "started. Loaded" in [message]  {
+    grok
+    {
+      match => { "message" => "\"%{GREEDYDATA:errorMessage}\"" }
+      add_tag => ["info"]
+    }
+  }
+
+  if "calls remain" in [message]  {
+    grok
+    {
+      match => { "message" => "\"%{GREEDYDATA:errorMessage}\"" }
+      add_tag => ["info"]
+    }
+  }
+
+  mutate {
+      rename => { "[message]" => "[original]"}
+  }
+}
+
+output {
+  elasticsearch {
+    id => "shhgit"
+    hosts => ["localhost:9200"]
+    index => "shhgit-%{+YYYY.MM.dd}"
+  }
+}