report.FillCveInfo - Code Metrics - Inspection of "feat(report): GitHub security alerts integration (..." - future-architect/vuls - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( 256c99...56d7d4 )

by kota

created 2019-02-20 03:04 UTC

report.FillCveInfo F

↳ Parent: report/report.go

Complexity

Conditions

Size

Total Lines	62
Code Lines	42

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	16
eloc	42
dl	0
loc	62
rs	2.4
c	0
b	0
f	0
nop	4

How to fix Long Method Complexity

/* Vuls - Vulnerability Scanner
Copyright (C) 2016  Future Corporation , Japan.

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

package report

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"reflect"
	"regexp"
	"sort"
	"strings"
	"time"

	"github.com/BurntSushi/toml"
	c "github.com/future-architect/vuls/config"
	"github.com/future-architect/vuls/contrib/owasp-dependency-check/parser"
	"github.com/future-architect/vuls/cwe"
	"github.com/future-architect/vuls/exploit"
	"github.com/future-architect/vuls/github"
	"github.com/future-architect/vuls/gost"
	"github.com/future-architect/vuls/models"
	"github.com/future-architect/vuls/oval"
	"github.com/future-architect/vuls/util"
	"github.com/hashicorp/uuid"
	gostdb "github.com/knqyf263/gost/db"
	cvedb "github.com/kotakanbe/go-cve-dictionary/db"
	ovaldb "github.com/kotakanbe/goval-dictionary/db"
	exploitdb "github.com/mozqnet/go-exploitdb/db"
)

const (
	vulsOpenTag  = "<vulsreport>"
	vulsCloseTag = "</vulsreport>"
)

// FillCveInfos fills CVE Detailed Information
func FillCveInfos(dbclient DBClient, rs []models.ScanResult, dir string) ([]models.ScanResult, error) {
	var filledResults []models.ScanResult
	reportedAt := time.Now()
	hostname, _ := os.Hostname()
	for _, r := range rs {
		if c.Conf.RefreshCve || needToRefreshCve(r) {
			if ovalSupported(&r) {
				r.ScannedCves = models.VulnInfos{}
			}
			cpeURIs := []string{}
			if len(r.Container.ContainerID) == 0 {
				cpeURIs = c.Conf.Servers[r.ServerName].CpeNames
				owaspDCXMLPath := c.Conf.Servers[r.ServerName].OwaspDCXMLPath
				if owaspDCXMLPath != "" {
					cpes, err := parser.Parse(owaspDCXMLPath)
					if err != nil {
						return nil, fmt.Errorf("Failed to read OWASP Dependency Check XML: %s, %s, %s",
							r.ServerName, owaspDCXMLPath, err)
					}
					cpeURIs = append(cpeURIs, cpes...)
				}
			} else {
				if s, ok := c.Conf.Servers[r.ServerName]; ok {
					if con, ok := s.Containers[r.Container.Name]; ok {
						cpeURIs = con.Cpes
						owaspDCXMLPath := con.OwaspDCXMLPath
						if owaspDCXMLPath != "" {
							cpes, err := parser.Parse(owaspDCXMLPath)
							if err != nil {
								return nil, fmt.Errorf("Failed to read OWASP Dependency Check XML: %s, %s, %s",
									r.ServerInfo(), owaspDCXMLPath, err)
							}
							cpeURIs = append(cpeURIs, cpes...)
						}
					}
				}
			}

			if err := FillCveInfo(dbclient, &r, cpeURIs); err != nil {
				return nil, err
			}
			r.Lang = c.Conf.Lang
			r.ReportedAt = reportedAt
			r.ReportedVersion = c.Version
			r.ReportedRevision = c.Revision
			r.ReportedBy = hostname
			r.Config.Report = c.Conf
			r.Config.Report.Servers = map[string]c.ServerInfo{
				r.ServerName: c.Conf.Servers[r.ServerName],
			}
			if err := overwriteJSONFile(dir, r); err != nil {
				return nil, fmt.Errorf("Failed to write JSON: %s", err)
			}
			filledResults = append(filledResults, r)
		} else {
			util.Log.Debugf("No need to refresh")
			filledResults = append(filledResults, r)
		}
	}

	if c.Conf.Diff {
		prevs, err := loadPrevious(filledResults)
		if err != nil {
			return nil, err
		}

		diff, err := diff(filledResults, prevs)
		if err != nil {
			return nil, err
		}
		filledResults = []models.ScanResult{}
		for _, r := range diff {
			if err := fillCveDetail(dbclient.CveDB, &r); err != nil {
				return nil, err
			}
			filledResults = append(filledResults, r)
		}
	}

	filtered := []models.ScanResult{}
	for _, r := range filledResults {
		r = r.FilterByCvssOver(c.Conf.CvssScoreOver)
		r = r.FilterIgnoreCves()
		r = r.FilterUnfixed()
		r = r.FilterIgnorePkgs()
		if c.Conf.IgnoreUnscoredCves {
			r.ScannedCves = r.ScannedCves.FindScoredVulns()
		}
		filtered = append(filtered, r)
	}
	return filtered, nil
}

// FillCveInfo fill scanResult with cve info.
func FillCveInfo(dbclient DBClient, r *models.ScanResult, cpeURIs []string, integrations ...c.IntegrationConf) error {
	util.Log.Debugf("need to refresh")

	nCVEs, err := FillWithOval(dbclient.OvalDB, r)
	if err != nil {
		return fmt.Errorf("Failed to fill with OVAL: %s", err)
	}
	util.Log.Infof("%s: %d CVEs are detected with OVAL",
		r.FormatServerName(), nCVEs)

	for i, v := range r.ScannedCves {
		for j, p := range v.AffectedPackages {
			if p.NotFixedYet && p.FixState == "" {
				p.FixState = "Not fixed yet"
				r.ScannedCves[i].AffectedPackages[j] = p
			}
		}
	}

	nCVEs, err = fillVulnByCpeURIs(dbclient.CveDB, r, cpeURIs)
	if err != nil {
		return fmt.Errorf("Failed to detect vulns of %s: %s", cpeURIs, err)
	}
	util.Log.Infof("%s: %d CVEs are detected with CPE", r.FormatServerName(), nCVEs)

	if len(integrations) != 0 {
		for k, v := range integrations[0].GitHubConf {
			c.Conf.Servers[r.ServerName].GitHubRepos[k] = v
		}
	}
	nCVEs, err = fillGitHubSecurityAlerts(r)
	if err != nil {
		return fmt.Errorf("Failed to access GitHub Security Alerts: %s", err)
	}
	util.Log.Infof("%s: %d CVEs are detected with GitHub Security Alerts", r.FormatServerName(), nCVEs)

	nCVEs, err = FillWithGost(dbclient.GostDB, r)
	if err != nil {
		return fmt.Errorf("Failed to fill with gost: %s", err)
	}
	util.Log.Infof("%s: %d unfixed CVEs are detected with gost",
		r.FormatServerName(), nCVEs)

	util.Log.Infof("Fill CVE detailed information with CVE-DB")
	if err := fillCveDetail(dbclient.CveDB, r); err != nil {
		return fmt.Errorf("Failed to fill with CVE: %s", err)
	}

	util.Log.Infof("Fill exploit information with Exploit-DB")
	nExploitCve, err := FillWithExploit(dbclient.ExploitDB, r)
	if err != nil {
		return fmt.Errorf("Failed to fill with exploit: %s", err)
	}
	util.Log.Infof("%s: %d exploits are detected",
		r.FormatServerName(), nExploitCve)

	enAlertCnt, jaAlertCnt := fillAlerts(r)
	util.Log.Infof("%s: en: %d, ja: %d alerts are detected",
		r.FormatServerName(), enAlertCnt, jaAlertCnt)

	fillCweDict(r)
	return nil
}

// fillCveDetail fetches NVD, JVN from CVE Database
func fillCveDetail(driver cvedb.DB, r *models.ScanResult) error {
	var cveIDs []string
	for _, v := range r.ScannedCves {
		cveIDs = append(cveIDs, v.CveID)
	}

	ds, err := CveClient.FetchCveDetails(driver, cveIDs)
	if err != nil {
		return err
	}
	for _, d := range ds {
		nvd := models.ConvertNvdJSONToModel(d.CveID, d.NvdJSON)
		if nvd == nil {
			nvd = models.ConvertNvdXMLToModel(d.CveID, d.NvdXML)
		}
		jvn := models.ConvertJvnToModel(d.CveID, d.Jvn)

		for cveID, vinfo := range r.ScannedCves {
			if vinfo.CveID == d.CveID {
				if vinfo.CveContents == nil {
					vinfo.CveContents = models.CveContents{}
				}
				for _, con := range []*models.CveContent{nvd, jvn} {
					if con != nil && !con.Empty() {
						vinfo.CveContents[con.Type] = *con
					}
				}
				r.ScannedCves[cveID] = vinfo
				break
			}
		}
	}
	return nil
}

// FillWithOval fetches OVAL database
func FillWithOval(driver ovaldb.DB, r *models.ScanResult) (nCVEs int, err error) {
	var ovalClient oval.Client
	var ovalFamily string

	switch r.Family {
	case c.Debian:
		ovalClient = oval.NewDebian()
		ovalFamily = c.Debian
	case c.Ubuntu:
		ovalClient = oval.NewUbuntu()
		ovalFamily = c.Ubuntu
	case c.RedHat:
		ovalClient = oval.NewRedhat()
		ovalFamily = c.RedHat
	case c.CentOS:
		ovalClient = oval.NewCentOS()
		//use RedHat's OVAL
		ovalFamily = c.RedHat
	case c.Oracle:
		ovalClient = oval.NewOracle()
		ovalFamily = c.Oracle
	case c.SUSEEnterpriseServer:
		// TODO other suse family
		ovalClient = oval.NewSUSE()
		ovalFamily = c.SUSEEnterpriseServer
	case c.Alpine:
		ovalClient = oval.NewAlpine()
		ovalFamily = c.Alpine
	case c.Amazon, c.Raspbian, c.FreeBSD, c.Windows:
		return 0, nil
	case c.ServerTypePseudo:
		return 0, nil
	default:
		if r.Family == "" {
			return 0, fmt.Errorf("Probably an error occurred during scanning. Check the error message")
		}
		return 0, fmt.Errorf("OVAL for %s is not implemented yet", r.Family)
	}

	if !c.Conf.OvalDict.IsFetchViaHTTP() {
		if driver == nil {
			return 0, nil
		}
		if err = driver.NewOvalDB(ovalFamily); err != nil {
			return 0, fmt.Errorf("Failed to New Oval DB. err: %s", err)
		}
	}

	util.Log.Debugf("Check whether oval fetched: %s %s", ovalFamily, r.Release)
	ok, err := ovalClient.CheckIfOvalFetched(driver, ovalFamily, r.Release)
	if err != nil {
		return 0, err
	}
	if !ok {
		util.Log.Warnf("OVAL entries of %s %s are not found. It's recommended to use OVAL to improve scanning accuracy. For details, see https://github.com/kotakanbe/goval-dictionary#usage , Then report with --ovaldb-path or --ovaldb-url flag", ovalFamily, r.Release)
		return 0, nil
	}

	_, err = ovalClient.CheckIfOvalFresh(driver, ovalFamily, r.Release)
	if err != nil {
		return 0, err
	}

	return ovalClient.FillWithOval(driver, r)
}

// FillWithGost fills CVEs with gost dataabase
// https://github.com/knqyf263/gost
func FillWithGost(driver gostdb.DB, r *models.ScanResult) (nCVEs int, err error) {
	gostClient := gost.NewClient(r.Family)
	// TODO chekc if fetched
	// TODO chekc if fresh enough
	return gostClient.FillWithGost(driver, r)
}

// FillWithExploit fills Exploits with exploit dataabase
// https://github.com/mozqnet/go-exploitdb
func FillWithExploit(driver exploitdb.DB, r *models.ScanResult) (nExploitCve int, err error) {
	// TODO chekc if fetched
	// TODO chekc if fresh enough
	return exploit.FillWithExploit(driver, r)
}

func fillVulnByCpeURIs(driver cvedb.DB, r *models.ScanResult, cpeURIs []string) (nCVEs int, err error) {
	for _, name := range cpeURIs {
		details, err := CveClient.FetchCveDetailsByCpeName(driver, name)
		if err != nil {
			return 0, err
		}
		for _, detail := range details {
			if val, ok := r.ScannedCves[detail.CveID]; ok {
				names := val.CpeURIs
				names = util.AppendIfMissing(names, name)
				val.CpeURIs = names
				val.Confidences.AppendIfMissing(models.CpeNameMatch)
				r.ScannedCves[detail.CveID] = val
			} else {
				v := models.VulnInfo{
					CveID:       detail.CveID,
					CpeURIs:     []string{name},
					Confidences: models.Confidences{models.CpeNameMatch},
				}
				r.ScannedCves[detail.CveID] = v
				nCVEs++
			}
		}
	}
	return nCVEs, nil
}

// https://help.github.com/articles/about-security-alerts-for-vulnerable-dependencies/
func fillGitHubSecurityAlerts(r *models.ScanResult) (nCVEs int, err error) {
	repos := c.Conf.Servers[r.ServerName].GitHubRepos
	for ownerRepo, setting := range repos {
		ss := strings.Split(ownerRepo, "/")
		owner, repo := ss[0], ss[1]
		n, err := github.FillGitHubSecurityAlerts(r, owner, repo, setting.Token)
		if err != nil {
			return 0, err
		}
		nCVEs += n
	}
	return nCVEs, nil
}

func fillCweDict(r *models.ScanResult) {
	uniqCweIDMap := map[string]bool{}
	for _, vinfo := range r.ScannedCves {
		for _, cont := range vinfo.CveContents {
			for _, id := range cont.CweIDs {
				if strings.HasPrefix(id, "CWE-") {
					id = strings.TrimPrefix(id, "CWE-")
					uniqCweIDMap[id] = true
				}
			}
		}
	}

	// TODO check the format of CWEID, clean CWEID
	// JVN, NVD XML, JSON, OVALs

	dict := map[string]models.CweDictEntry{}
	for id := range uniqCweIDMap {
		entry := models.CweDictEntry{}
		if e, ok := cwe.CweDictEn[id]; ok {
			if rank, ok := cwe.OwaspTopTen2017[id]; ok {
				entry.OwaspTopTen2017 = rank
			}
			entry.En = &e
		} else {
			util.Log.Debugf("CWE-ID %s is not found in English CWE Dict", id)
			entry.En = &cwe.Cwe{CweID: id}
		}

		if c.Conf.Lang == "ja" {
			if e, ok := cwe.CweDictJa[id]; ok {
				if rank, ok := cwe.OwaspTopTen2017[id]; ok {
					entry.OwaspTopTen2017 = rank
				}
				entry.Ja = &e
			} else {
				util.Log.Debugf("CWE-ID %s is not found in Japanese CWE Dict", id)
				entry.Ja = &cwe.Cwe{CweID: id}
			}
		}
		dict[id] = entry
	}
	r.CweDict = dict
	return
}

func fillAlerts(r *models.ScanResult) (enCnt int, jaCnt int) {
	for cveID, vuln := range r.ScannedCves {
		enAs, jaAs := models.GetAlertsByCveID(cveID, "en"), models.GetAlertsByCveID(cveID, "ja")
		vuln.AlertDict = models.AlertDict{
			Ja: jaAs,
			En: enAs,
		}
		r.ScannedCves[cveID] = vuln
		enCnt += len(enAs)
		jaCnt += len(jaAs)
	}
	return enCnt, jaCnt
}

const reUUID = "[\\da-f]{8}-[\\da-f]{4}-[\\da-f]{4}-[\\da-f]{4}-[\\da-f]{12}"

// EnsureUUIDs generate a new UUID of the scan target server if UUID is not assigned yet.
// And then set the generated UUID to config.toml and scan results.
func EnsureUUIDs(configPath string, results models.ScanResults) error {
	// Sort Host->Container
	sort.Slice(results, func(i, j int) bool {
		if results[i].ServerName == results[j].ServerName {
			return results[i].Container.ContainerID < results[j].Container.ContainerID
		}
		return results[i].ServerName < results[j].ServerName
	})

	for i, r := range results {
		server := c.Conf.Servers[r.ServerName]
		if server.UUIDs == nil {
			server.UUIDs = map[string]string{}
		}

		name := ""
		if r.IsContainer() {
			name = fmt.Sprintf("%s@%s", r.Container.Name, r.ServerName)

			// Scanning with the -containers-only flag at scan time, the UUID of Container Host may not be generated,
			// so check it. Otherwise create a UUID of the Container Host and set it.
			serverUUID := ""
			if id, ok := server.UUIDs[r.ServerName]; !ok {
				serverUUID = uuid.GenerateUUID()
			} else {
				matched, err := regexp.MatchString(reUUID, id)
				if !matched || err != nil {
					serverUUID = uuid.GenerateUUID()
				}
			}
			if serverUUID != "" {
				server.UUIDs[r.ServerName] = serverUUID
			}
		} else {
			name = r.ServerName
		}

		if id, ok := server.UUIDs[name]; ok {
			matched, err := regexp.MatchString(reUUID, id)
			if !matched || err != nil {
				util.Log.Warnf("UUID is invalid. Re-generate UUID %s: %s", id, err)
			} else {
				if r.IsContainer() {
					results[i].Container.UUID = id
					results[i].ServerUUID = server.UUIDs[r.ServerName]
				} else {
					results[i].ServerUUID = id
				}
				// continue if the UUID has already assigned and valid
				continue
			}
		}

		// Generate a new UUID and set to config and scan result
		id := uuid.GenerateUUID()
		server.UUIDs[name] = id
		server = cleanForTOMLEncoding(server, c.Conf.Default)
		c.Conf.Servers[r.ServerName] = server

		if r.IsContainer() {
			results[i].Container.UUID = id
			results[i].ServerUUID = server.UUIDs[r.ServerName]
		} else {
			results[i].ServerUUID = id
		}
	}

	for name, server := range c.Conf.Servers {
		server = cleanForTOMLEncoding(server, c.Conf.Default)
		c.Conf.Servers[name] = server
	}

	email := &c.Conf.EMail
	if email.SMTPAddr == "" {
		email = nil
	}

	slack := &c.Conf.Slack
	if slack.HookURL == "" {
		slack = nil
	}

	cveDict := &c.Conf.CveDict
	ovalDict := &c.Conf.OvalDict
	gost := &c.Conf.Gost
	exploit := &c.Conf.Exploit
	http := &c.Conf.HTTP
	if http.URL == "" {
		http = nil
	}

	syslog := &c.Conf.Syslog
	if syslog.Host == "" {
		syslog = nil
	}

	aws := &c.Conf.AWS
	if aws.S3Bucket == "" {
		aws = nil
	}

	azure := &c.Conf.Azure
	if azure.AccountName == "" {
		azure = nil
	}

	stride := &c.Conf.Stride
	if stride.HookURL == "" {
		stride = nil
	}

	hipChat := &c.Conf.HipChat
	if hipChat.AuthToken == "" {
		hipChat = nil
	}

	chatWork := &c.Conf.ChatWork
	if chatWork.APIToken == "" {
		chatWork = nil
	}

	saas := &c.Conf.Saas
	if saas.GroupID == 0 {
		saas = nil
	}

	c := struct {
		CveDict  *c.GoCveDictConf `toml:"cveDict"`
		OvalDict *c.GovalDictConf `toml:"ovalDict"`
		Gost     *c.GostConf      `toml:"gost"`
		Exploit  *c.ExploitConf   `toml:"exploit"`
		Slack    *c.SlackConf     `toml:"slack"`
		Email    *c.SMTPConf      `toml:"email"`
		HTTP     *c.HTTPConf      `toml:"http"`
		Syslog   *c.SyslogConf    `toml:"syslog"`
		AWS      *c.AWS           `toml:"aws"`
		Azure    *c.Azure         `toml:"azure"`
		Stride   *c.StrideConf    `toml:"stride"`
		HipChat  *c.HipChatConf   `toml:"hipChat"`
		ChatWork *c.ChatWorkConf  `toml:"chatWork"`
		Saas     *c.SaasConf      `toml:"saas"`

		Default c.ServerInfo            `toml:"default"`
		Servers map[string]c.ServerInfo `toml:"servers"`
	}{
		CveDict:  cveDict,
		OvalDict: ovalDict,
		Gost:     gost,
		Exploit:  exploit,
		Slack:    slack,
		Email:    email,
		HTTP:     http,
		Syslog:   syslog,
		AWS:      aws,
		Azure:    azure,
		Stride:   stride,
		HipChat:  hipChat,
		ChatWork: chatWork,
		Saas:     saas,

		Default: c.Conf.Default,
		Servers: c.Conf.Servers,
	}

	// rename the current config.toml to config.toml.bak
	info, err := os.Lstat(configPath)
	if err != nil {
		return fmt.Errorf("Failed to lstat %s: %s", configPath, err)
	}
	realPath := configPath
	if info.Mode()&os.ModeSymlink == os.ModeSymlink {
		if realPath, err = os.Readlink(configPath); err != nil {
			return fmt.Errorf("Failed to Read link %s: %s", configPath, err)
		}
	}
	if err := os.Rename(realPath, realPath+".bak"); err != nil {
		return fmt.Errorf("Failed to rename %s: %s", configPath, err)
	}

	var buf bytes.Buffer
	if err := toml.NewEncoder(&buf).Encode(c); err != nil {
		return fmt.Errorf("Failed to encode to toml: %s", err)
	}
	str := strings.Replace(buf.String(), "\n  [", "\n\n  [", -1)
	str = fmt.Sprintf("%s\n\n%s",
		"# See REAME for details: https://vuls.io/docs/en/usage-settings.html",
		str)

	return ioutil.WriteFile(realPath, []byte(str), 0600)
}

func cleanForTOMLEncoding(server c.ServerInfo, def c.ServerInfo) c.ServerInfo {
	if reflect.DeepEqual(server.Optional, def.Optional) {
		server.Optional = nil
	}

	if def.User == server.User {
		server.User = ""
	}

	if def.Host == server.Host {
		server.Host = ""
	}

	if def.Port == server.Port {
		server.Port = ""
	}

	if def.KeyPath == server.KeyPath {
		server.KeyPath = ""
	}

	if reflect.DeepEqual(server.ScanMode, def.ScanMode) {
		server.ScanMode = nil
	}

	if def.Type == server.Type {
		server.Type = ""
	}

	if reflect.DeepEqual(server.CpeNames, def.CpeNames) {
		server.CpeNames = nil
	}

	if def.OwaspDCXMLPath == server.OwaspDCXMLPath {
		server.OwaspDCXMLPath = ""
	}

	if reflect.DeepEqual(server.IgnoreCves, def.IgnoreCves) {
		server.IgnoreCves = nil
	}

	if reflect.DeepEqual(server.Enablerepo, def.Enablerepo) {
		server.Enablerepo = nil
	}

	for k, v := range def.Optional {
		if vv, ok := server.Optional[k]; ok && v == vv {
			delete(server.Optional, k)
		}
	}

	return server
}


1			/* Vuls - Vulnerability Scanner
2			Copyright (C) 2016 Future Corporation , Japan.
3
4			This program is free software: you can redistribute it and/or modify
5			it under the terms of the GNU General Public License as published by
6			the Free Software Foundation, either version 3 of the License, or
7			(at your option) any later version.
8
9			This program is distributed in the hope that it will be useful,
10			but WITHOUT ANY WARRANTY; without even the implied warranty of
11			MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12			GNU General Public License for more details.
13
14			You should have received a copy of the GNU General Public License
15			along with this program. If not, see <http://www.gnu.org/licenses/>.
16			*/
17
18			package report
19
20			import (
21			"bytes"
22			"fmt"
23			"io/ioutil"
24			"os"
25			"reflect"
26			"regexp"
27			"sort"
28			"strings"
29			"time"
30
31			"github.com/BurntSushi/toml"
32			c "github.com/future-architect/vuls/config"
33			"github.com/future-architect/vuls/contrib/owasp-dependency-check/parser"
34			"github.com/future-architect/vuls/cwe"
35			"github.com/future-architect/vuls/exploit"
36			"github.com/future-architect/vuls/github"
37			"github.com/future-architect/vuls/gost"
38			"github.com/future-architect/vuls/models"
39			"github.com/future-architect/vuls/oval"
40			"github.com/future-architect/vuls/util"
41			"github.com/hashicorp/uuid"
42			gostdb "github.com/knqyf263/gost/db"
43			cvedb "github.com/kotakanbe/go-cve-dictionary/db"
44			ovaldb "github.com/kotakanbe/goval-dictionary/db"
45			exploitdb "github.com/mozqnet/go-exploitdb/db"
46			)
47
48			const (
49			vulsOpenTag = "<vulsreport>"
50			vulsCloseTag = "</vulsreport>"
51			)
52
53			// FillCveInfos fills CVE Detailed Information
54			func FillCveInfos(dbclient DBClient, rs []models.ScanResult, dir string) ([]models.ScanResult, error) {
55			var filledResults []models.ScanResult
56			reportedAt := time.Now()
57			hostname, _ := os.Hostname()
58			for _, r := range rs {
59			if c.Conf.RefreshCve \|\| needToRefreshCve(r) {
60			if ovalSupported(&r) {
61			r.ScannedCves = models.VulnInfos{}
62			}
63			cpeURIs := []string{}
64			if len(r.Container.ContainerID) == 0 {
65			cpeURIs = c.Conf.Servers[r.ServerName].CpeNames
66			owaspDCXMLPath := c.Conf.Servers[r.ServerName].OwaspDCXMLPath
67			if owaspDCXMLPath != "" {
68			cpes, err := parser.Parse(owaspDCXMLPath)
69			if err != nil {
70			return nil, fmt.Errorf("Failed to read OWASP Dependency Check XML: %s, %s, %s",
71			r.ServerName, owaspDCXMLPath, err)
72			}
73			cpeURIs = append(cpeURIs, cpes...)
74			}
75			} else {
76			if s, ok := c.Conf.Servers[r.ServerName]; ok {
77			if con, ok := s.Containers[r.Container.Name]; ok {
78			cpeURIs = con.Cpes
79			owaspDCXMLPath := con.OwaspDCXMLPath
80			if owaspDCXMLPath != "" {
81			cpes, err := parser.Parse(owaspDCXMLPath)
82			if err != nil {
83			return nil, fmt.Errorf("Failed to read OWASP Dependency Check XML: %s, %s, %s",
84			r.ServerInfo(), owaspDCXMLPath, err)
85			}
86			cpeURIs = append(cpeURIs, cpes...)
87			}
88			}
89			}
90			}
91
92			if err := FillCveInfo(dbclient, &r, cpeURIs); err != nil {
93			return nil, err
94			}
95			r.Lang = c.Conf.Lang
96			r.ReportedAt = reportedAt
97			r.ReportedVersion = c.Version
98			r.ReportedRevision = c.Revision
99			r.ReportedBy = hostname
100			r.Config.Report = c.Conf
101			r.Config.Report.Servers = map[string]c.ServerInfo{
102			r.ServerName: c.Conf.Servers[r.ServerName],
103			}
104			if err := overwriteJSONFile(dir, r); err != nil {
105			return nil, fmt.Errorf("Failed to write JSON: %s", err)
106			}
107			filledResults = append(filledResults, r)
108			} else {
109			util.Log.Debugf("No need to refresh")
110			filledResults = append(filledResults, r)
111			}
112			}
113
114			if c.Conf.Diff {
115			prevs, err := loadPrevious(filledResults)
116			if err != nil {
117			return nil, err
118			}
119
120			diff, err := diff(filledResults, prevs)
121			if err != nil {
122			return nil, err
123			}
124			filledResults = []models.ScanResult{}
125			for _, r := range diff {
126			if err := fillCveDetail(dbclient.CveDB, &r); err != nil {
127			return nil, err
128			}
129			filledResults = append(filledResults, r)
130			}
131			}
132
133			filtered := []models.ScanResult{}
134			for _, r := range filledResults {
135			r = r.FilterByCvssOver(c.Conf.CvssScoreOver)
136			r = r.FilterIgnoreCves()
137			r = r.FilterUnfixed()
138			r = r.FilterIgnorePkgs()
139			if c.Conf.IgnoreUnscoredCves {
140			r.ScannedCves = r.ScannedCves.FindScoredVulns()
141			}
142			filtered = append(filtered, r)
143			}
144			return filtered, nil
145			}
146
147			// FillCveInfo fill scanResult with cve info.
148			func FillCveInfo(dbclient DBClient, r *models.ScanResult, cpeURIs []string, integrations ...c.IntegrationConf) error {
149			util.Log.Debugf("need to refresh")
150
151			nCVEs, err := FillWithOval(dbclient.OvalDB, r)
152			if err != nil {
153			return fmt.Errorf("Failed to fill with OVAL: %s", err)
154			}
155			util.Log.Infof("%s: %d CVEs are detected with OVAL",
156			r.FormatServerName(), nCVEs)
157
158			for i, v := range r.ScannedCves {
159			for j, p := range v.AffectedPackages {
160			if p.NotFixedYet && p.FixState == "" {
161			p.FixState = "Not fixed yet"
162			r.ScannedCves[i].AffectedPackages[j] = p
163			}
164			}
165			}
166
167			nCVEs, err = fillVulnByCpeURIs(dbclient.CveDB, r, cpeURIs)
168			if err != nil {
169			return fmt.Errorf("Failed to detect vulns of %s: %s", cpeURIs, err)
170			}
171			util.Log.Infof("%s: %d CVEs are detected with CPE", r.FormatServerName(), nCVEs)
172
173			if len(integrations) != 0 {
174			for k, v := range integrations[0].GitHubConf {
175			c.Conf.Servers[r.ServerName].GitHubRepos[k] = v
176			}
177			}
178			nCVEs, err = fillGitHubSecurityAlerts(r)
179			if err != nil {
180			return fmt.Errorf("Failed to access GitHub Security Alerts: %s", err)
181			}
182			util.Log.Infof("%s: %d CVEs are detected with GitHub Security Alerts", r.FormatServerName(), nCVEs)
183
184			nCVEs, err = FillWithGost(dbclient.GostDB, r)
185			if err != nil {
186			return fmt.Errorf("Failed to fill with gost: %s", err)
187			}
188			util.Log.Infof("%s: %d unfixed CVEs are detected with gost",
189			r.FormatServerName(), nCVEs)
190
191			util.Log.Infof("Fill CVE detailed information with CVE-DB")
192			if err := fillCveDetail(dbclient.CveDB, r); err != nil {
193			return fmt.Errorf("Failed to fill with CVE: %s", err)
194			}
195
196			util.Log.Infof("Fill exploit information with Exploit-DB")
197			nExploitCve, err := FillWithExploit(dbclient.ExploitDB, r)
198			if err != nil {
199			return fmt.Errorf("Failed to fill with exploit: %s", err)
200			}
201			util.Log.Infof("%s: %d exploits are detected",
202			r.FormatServerName(), nExploitCve)
203
204			enAlertCnt, jaAlertCnt := fillAlerts(r)
205			util.Log.Infof("%s: en: %d, ja: %d alerts are detected",
206			r.FormatServerName(), enAlertCnt, jaAlertCnt)
207
208			fillCweDict(r)
209			return nil
210			}
211
212			// fillCveDetail fetches NVD, JVN from CVE Database
213			func fillCveDetail(driver cvedb.DB, r *models.ScanResult) error {
214			var cveIDs []string
215			for _, v := range r.ScannedCves {
216			cveIDs = append(cveIDs, v.CveID)
217			}
218
219			ds, err := CveClient.FetchCveDetails(driver, cveIDs)
220			if err != nil {
221			return err
222			}
223			for _, d := range ds {
224			nvd := models.ConvertNvdJSONToModel(d.CveID, d.NvdJSON)
225			if nvd == nil {
226			nvd = models.ConvertNvdXMLToModel(d.CveID, d.NvdXML)
227			}
228			jvn := models.ConvertJvnToModel(d.CveID, d.Jvn)
229
230			for cveID, vinfo := range r.ScannedCves {
231			if vinfo.CveID == d.CveID {
232			if vinfo.CveContents == nil {
233			vinfo.CveContents = models.CveContents{}
234			}
235			for _, con := range []*models.CveContent{nvd, jvn} {
236			if con != nil && !con.Empty() {
237			vinfo.CveContents[con.Type] = *con
238			}
239			}
240			r.ScannedCves[cveID] = vinfo
241			break
242			}
243			}
244			}
245			return nil
246			}
247
248			// FillWithOval fetches OVAL database
249			func FillWithOval(driver ovaldb.DB, r *models.ScanResult) (nCVEs int, err error) {
250			var ovalClient oval.Client
251			var ovalFamily string
252
253			switch r.Family {
254			case c.Debian:
255			ovalClient = oval.NewDebian()
256			ovalFamily = c.Debian
257			case c.Ubuntu:
258			ovalClient = oval.NewUbuntu()
259			ovalFamily = c.Ubuntu
260			case c.RedHat:
261			ovalClient = oval.NewRedhat()
262			ovalFamily = c.RedHat
263			case c.CentOS:
264			ovalClient = oval.NewCentOS()
265			//use RedHat's OVAL
266			ovalFamily = c.RedHat
267			case c.Oracle:
268			ovalClient = oval.NewOracle()
269			ovalFamily = c.Oracle
270			case c.SUSEEnterpriseServer:
271			// TODO other suse family
272			ovalClient = oval.NewSUSE()
273			ovalFamily = c.SUSEEnterpriseServer
274			case c.Alpine:
275			ovalClient = oval.NewAlpine()
276			ovalFamily = c.Alpine
277			case c.Amazon, c.Raspbian, c.FreeBSD, c.Windows:
278			return 0, nil
279			case c.ServerTypePseudo:
280			return 0, nil
281			default:
282			if r.Family == "" {
283			return 0, fmt.Errorf("Probably an error occurred during scanning. Check the error message")
284			}
285			return 0, fmt.Errorf("OVAL for %s is not implemented yet", r.Family)
286			}
287
288			if !c.Conf.OvalDict.IsFetchViaHTTP() {
289			if driver == nil {
290			return 0, nil
291			}
292			if err = driver.NewOvalDB(ovalFamily); err != nil {
293			return 0, fmt.Errorf("Failed to New Oval DB. err: %s", err)
294			}
295			}
296
297			util.Log.Debugf("Check whether oval fetched: %s %s", ovalFamily, r.Release)
298			ok, err := ovalClient.CheckIfOvalFetched(driver, ovalFamily, r.Release)
299			if err != nil {
300			return 0, err
301			}
302			if !ok {
303			util.Log.Warnf("OVAL entries of %s %s are not found. It's recommended to use OVAL to improve scanning accuracy. For details, see https://github.com/kotakanbe/goval-dictionary#usage , Then report with --ovaldb-path or --ovaldb-url flag", ovalFamily, r.Release)
304			return 0, nil
305			}
306
307			_, err = ovalClient.CheckIfOvalFresh(driver, ovalFamily, r.Release)
308			if err != nil {
309			return 0, err
310			}
311
312			return ovalClient.FillWithOval(driver, r)
313			}
314
315			// FillWithGost fills CVEs with gost dataabase
316			// https://github.com/knqyf263/gost
317			func FillWithGost(driver gostdb.DB, r *models.ScanResult) (nCVEs int, err error) {
318			gostClient := gost.NewClient(r.Family)
319			// TODO chekc if fetched
320			// TODO chekc if fresh enough
321			return gostClient.FillWithGost(driver, r)
322			}
323
324			// FillWithExploit fills Exploits with exploit dataabase
325			// https://github.com/mozqnet/go-exploitdb
326			func FillWithExploit(driver exploitdb.DB, r *models.ScanResult) (nExploitCve int, err error) {
327			// TODO chekc if fetched
328			// TODO chekc if fresh enough
329			return exploit.FillWithExploit(driver, r)
330			}
331
332			func fillVulnByCpeURIs(driver cvedb.DB, r *models.ScanResult, cpeURIs []string) (nCVEs int, err error) {
333			for _, name := range cpeURIs {
334			details, err := CveClient.FetchCveDetailsByCpeName(driver, name)
335			if err != nil {
336			return 0, err
337			}
338			for _, detail := range details {
339			if val, ok := r.ScannedCves[detail.CveID]; ok {
340			names := val.CpeURIs
341			names = util.AppendIfMissing(names, name)
342			val.CpeURIs = names
343			val.Confidences.AppendIfMissing(models.CpeNameMatch)
344			r.ScannedCves[detail.CveID] = val
345			} else {
346			v := models.VulnInfo{
347			CveID: detail.CveID,
348			CpeURIs: []string{name},
349			Confidences: models.Confidences{models.CpeNameMatch},
350			}
351			r.ScannedCves[detail.CveID] = v
352			nCVEs++
353			}
354			}
355			}
356			return nCVEs, nil
357			}
358
359			// https://help.github.com/articles/about-security-alerts-for-vulnerable-dependencies/
360			func fillGitHubSecurityAlerts(r *models.ScanResult) (nCVEs int, err error) {
361			repos := c.Conf.Servers[r.ServerName].GitHubRepos
362			for ownerRepo, setting := range repos {
363			ss := strings.Split(ownerRepo, "/")
364			owner, repo := ss[0], ss[1]
365			n, err := github.FillGitHubSecurityAlerts(r, owner, repo, setting.Token)
366			if err != nil {
367			return 0, err
368			}
369			nCVEs += n
370			}
371			return nCVEs, nil
372			}
373
374			func fillCweDict(r *models.ScanResult) {
375			uniqCweIDMap := map[string]bool{}
376			for _, vinfo := range r.ScannedCves {
377			for _, cont := range vinfo.CveContents {
378			for _, id := range cont.CweIDs {
379			if strings.HasPrefix(id, "CWE-") {
380			id = strings.TrimPrefix(id, "CWE-")
381			uniqCweIDMap[id] = true
382			}
383			}
384			}
385			}
386
387			// TODO check the format of CWEID, clean CWEID
388			// JVN, NVD XML, JSON, OVALs
389
390			dict := map[string]models.CweDictEntry{}
391			for id := range uniqCweIDMap {
392			entry := models.CweDictEntry{}
393			if e, ok := cwe.CweDictEn[id]; ok {
394			if rank, ok := cwe.OwaspTopTen2017[id]; ok {
395			entry.OwaspTopTen2017 = rank
396			}
397			entry.En = &e
398			} else {
399			util.Log.Debugf("CWE-ID %s is not found in English CWE Dict", id)
400			entry.En = &cwe.Cwe{CweID: id}
401			}
402
403			if c.Conf.Lang == "ja" {
404			if e, ok := cwe.CweDictJa[id]; ok {
405			if rank, ok := cwe.OwaspTopTen2017[id]; ok {
406			entry.OwaspTopTen2017 = rank
407			}
408			entry.Ja = &e
409			} else {
410			util.Log.Debugf("CWE-ID %s is not found in Japanese CWE Dict", id)
411			entry.Ja = &cwe.Cwe{CweID: id}
412			}
413			}
414			dict[id] = entry
415			}
416			r.CweDict = dict
417			return
418			}
419
420			func fillAlerts(r *models.ScanResult) (enCnt int, jaCnt int) {
421			for cveID, vuln := range r.ScannedCves {
422			enAs, jaAs := models.GetAlertsByCveID(cveID, "en"), models.GetAlertsByCveID(cveID, "ja")
423			vuln.AlertDict = models.AlertDict{
424			Ja: jaAs,
425			En: enAs,
426			}
427			r.ScannedCves[cveID] = vuln
428			enCnt += len(enAs)
429			jaCnt += len(jaAs)
430			}
431			return enCnt, jaCnt
432			}
433
434			const reUUID = "[\\da-f]{8}-[\\da-f]{4}-[\\da-f]{4}-[\\da-f]{4}-[\\da-f]{12}"
435
436			// EnsureUUIDs generate a new UUID of the scan target server if UUID is not assigned yet.
437			// And then set the generated UUID to config.toml and scan results.
438			func EnsureUUIDs(configPath string, results models.ScanResults) error {
439			// Sort Host->Container
440			sort.Slice(results, func(i, j int) bool {
441			if results[i].ServerName == results[j].ServerName {
442			return results[i].Container.ContainerID < results[j].Container.ContainerID
443			}
444			return results[i].ServerName < results[j].ServerName
445			})
446
447			for i, r := range results {
448			server := c.Conf.Servers[r.ServerName]
449			if server.UUIDs == nil {
450			server.UUIDs = map[string]string{}
451			}
452
453			name := ""
454			if r.IsContainer() {
455			name = fmt.Sprintf("%s@%s", r.Container.Name, r.ServerName)
456
457			// Scanning with the -containers-only flag at scan time, the UUID of Container Host may not be generated,
458			// so check it. Otherwise create a UUID of the Container Host and set it.
459			serverUUID := ""
460			if id, ok := server.UUIDs[r.ServerName]; !ok {
461			serverUUID = uuid.GenerateUUID()
462			} else {
463			matched, err := regexp.MatchString(reUUID, id)
464			if !matched \|\| err != nil {
465			serverUUID = uuid.GenerateUUID()
466			}
467			}
468			if serverUUID != "" {
469			server.UUIDs[r.ServerName] = serverUUID
470			}
471			} else {
472			name = r.ServerName
473			}
474
475			if id, ok := server.UUIDs[name]; ok {
476			matched, err := regexp.MatchString(reUUID, id)
477			if !matched \|\| err != nil {
478			util.Log.Warnf("UUID is invalid. Re-generate UUID %s: %s", id, err)
479			} else {
480			if r.IsContainer() {
481			results[i].Container.UUID = id
482			results[i].ServerUUID = server.UUIDs[r.ServerName]
483			} else {
484			results[i].ServerUUID = id
485			}
486			// continue if the UUID has already assigned and valid
487			continue
488			}
489			}
490
491			// Generate a new UUID and set to config and scan result
492			id := uuid.GenerateUUID()
493			server.UUIDs[name] = id
494			server = cleanForTOMLEncoding(server, c.Conf.Default)
495			c.Conf.Servers[r.ServerName] = server
496
497			if r.IsContainer() {
498			results[i].Container.UUID = id
499			results[i].ServerUUID = server.UUIDs[r.ServerName]
500			} else {
501			results[i].ServerUUID = id
502			}
503			}
504
505			for name, server := range c.Conf.Servers {
506			server = cleanForTOMLEncoding(server, c.Conf.Default)
507			c.Conf.Servers[name] = server
508			}
509
510			email := &c.Conf.EMail
511			if email.SMTPAddr == "" {
512			email = nil
513			}
514
515			slack := &c.Conf.Slack
516			if slack.HookURL == "" {
517			slack = nil
518			}
519
520			cveDict := &c.Conf.CveDict
521			ovalDict := &c.Conf.OvalDict
522			gost := &c.Conf.Gost
523			exploit := &c.Conf.Exploit
524			http := &c.Conf.HTTP
525			if http.URL == "" {
526			http = nil
527			}
528
529			syslog := &c.Conf.Syslog
530			if syslog.Host == "" {
531			syslog = nil
532			}
533
534			aws := &c.Conf.AWS
535			if aws.S3Bucket == "" {
536			aws = nil
537			}
538
539			azure := &c.Conf.Azure
540			if azure.AccountName == "" {
541			azure = nil
542			}
543
544			stride := &c.Conf.Stride
545			if stride.HookURL == "" {
546			stride = nil
547			}
548
549			hipChat := &c.Conf.HipChat
550			if hipChat.AuthToken == "" {
551			hipChat = nil
552			}
553
554			chatWork := &c.Conf.ChatWork
555			if chatWork.APIToken == "" {
556			chatWork = nil
557			}
558
559			saas := &c.Conf.Saas
560			if saas.GroupID == 0 {
561			saas = nil
562			}
563
564			c := struct {
565			CveDict *c.GoCveDictConf `toml:"cveDict"`
566			OvalDict *c.GovalDictConf `toml:"ovalDict"`
567			Gost *c.GostConf `toml:"gost"`
568			Exploit *c.ExploitConf `toml:"exploit"`
569			Slack *c.SlackConf `toml:"slack"`
570			Email *c.SMTPConf `toml:"email"`
571			HTTP *c.HTTPConf `toml:"http"`
572			Syslog *c.SyslogConf `toml:"syslog"`
573			AWS *c.AWS `toml:"aws"`
574			Azure *c.Azure `toml:"azure"`
575			Stride *c.StrideConf `toml:"stride"`
576			HipChat *c.HipChatConf `toml:"hipChat"`
577			ChatWork *c.ChatWorkConf `toml:"chatWork"`
578			Saas *c.SaasConf `toml:"saas"`
579
580			Default c.ServerInfo `toml:"default"`
581			Servers map[string]c.ServerInfo `toml:"servers"`
582			}{
583			CveDict: cveDict,
584			OvalDict: ovalDict,
585			Gost: gost,
586			Exploit: exploit,
587			Slack: slack,
588			Email: email,
589			HTTP: http,
590			Syslog: syslog,
591			AWS: aws,
592			Azure: azure,
593			Stride: stride,
594			HipChat: hipChat,
595			ChatWork: chatWork,
596			Saas: saas,
597
598			Default: c.Conf.Default,
599			Servers: c.Conf.Servers,
600			}
601
602			// rename the current config.toml to config.toml.bak
603			info, err := os.Lstat(configPath)
604			if err != nil {
605			return fmt.Errorf("Failed to lstat %s: %s", configPath, err)
606			}
607			realPath := configPath
608			if info.Mode()&os.ModeSymlink == os.ModeSymlink {
609			if realPath, err = os.Readlink(configPath); err != nil {
610			return fmt.Errorf("Failed to Read link %s: %s", configPath, err)
611			}
612			}
613			if err := os.Rename(realPath, realPath+".bak"); err != nil {
614			return fmt.Errorf("Failed to rename %s: %s", configPath, err)
615			}
616
617			var buf bytes.Buffer
618			if err := toml.NewEncoder(&buf).Encode(c); err != nil {
619			return fmt.Errorf("Failed to encode to toml: %s", err)
620			}
621			str := strings.Replace(buf.String(), "\n [", "\n\n [", -1)
622			str = fmt.Sprintf("%s\n\n%s",
623			"# See REAME for details: https://vuls.io/docs/en/usage-settings.html",
624			str)
625
626			return ioutil.WriteFile(realPath, []byte(str), 0600)
627			}
628
629			func cleanForTOMLEncoding(server c.ServerInfo, def c.ServerInfo) c.ServerInfo {
630			if reflect.DeepEqual(server.Optional, def.Optional) {
631			server.Optional = nil
632			}
633
634			if def.User == server.User {
635			server.User = ""
636			}
637
638			if def.Host == server.Host {
639			server.Host = ""
640			}
641
642			if def.Port == server.Port {
643			server.Port = ""
644			}
645
646			if def.KeyPath == server.KeyPath {
647			server.KeyPath = ""
648			}
649
650			if reflect.DeepEqual(server.ScanMode, def.ScanMode) {
651			server.ScanMode = nil
652			}
653
654			if def.Type == server.Type {
655			server.Type = ""
656			}
657
658			if reflect.DeepEqual(server.CpeNames, def.CpeNames) {
659			server.CpeNames = nil
660			}
661
662			if def.OwaspDCXMLPath == server.OwaspDCXMLPath {
663			server.OwaspDCXMLPath = ""
664			}
665
666			if reflect.DeepEqual(server.IgnoreCves, def.IgnoreCves) {
667			server.IgnoreCves = nil
668			}
669
670			if reflect.DeepEqual(server.Enablerepo, def.Enablerepo) {
671			server.Enablerepo = nil
672			}
673
674			for k, v := range def.Optional {
675			if vv, ok := server.Optional[k]; ok && v == vv {
676			delete(server.Optional, k)
677			}
678			}
679
680			return server
681			}
682

future-architect / vuls

Push — master ( 256c99...56d7d4 )

report.FillCveInfo F

Complexity

Size

Duplication

Importance

How to fix Long Method Complexity

Long Method

Complexity

Duplication Side-by-Side

Filter issues like