Skip to content

Commit

Permalink
Add per-directory file counts in the filecount input (#4752)
Browse files Browse the repository at this point in the history
  • Loading branch information
@Jaeyo @danielnelson
Jaeyo authored and danielnelson committed Oct 5, 2018
1 parent f81696b commit 030f944
Show file tree
Hide file tree
Showing 2 changed files with 141 additions and 55 deletions.
101 changes: 75 additions & 26 deletions plugins/inputs/filecount/filecount.go
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,30 @@
package filecount

import (
"fmt"
"os"
"path/filepath"
"time"

"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/internal"
"github.com/influxdata/telegraf/internal/globpath"
"github.com/influxdata/telegraf/plugins/inputs"
)

const sampleConfig = `
## Directory to gather stats about.
## deprecated in 1.9; use the directories option
directory = "/var/cache/apt/archives"
## Directories to gather stats about.
## This accept standard unit glob matching rules, but with the addition of
## ** as a "super asterisk". ie:
## /var/log/** -> recursively find all directories in /var/log and count files in each directories
## /var/log/*/* -> find all directories with a parent dir in /var/log and count files in each directories
## /var/log -> count all files in /var/log and all of its subdirectories
directories = ["/var/cache/apt/archives"]
## Only count files that match the name pattern. Defaults to "*".
name = "*.deb"
Expand All @@ -35,7 +46,8 @@ const sampleConfig = `
`

type FileCount struct {
Directory string
Directory string // deprecated in 1.9
Directories []string
Name string
Recursive bool
RegularOnly bool
Expand All @@ -44,7 +56,6 @@ type FileCount struct {
fileFilters []fileFilterFunc
}

type countFunc func(os.FileInfo)
type fileFilterFunc func(os.FileInfo) (bool, error)

func (_ *FileCount) Description() string {
Expand Down Expand Up @@ -125,18 +136,40 @@ func absDuration(x time.Duration) time.Duration {
return x
}

func count(basedir string, recursive bool, countFn countFunc) error {
func (fc *FileCount) count(acc telegraf.Accumulator, basedir string, recursive bool) {
numFiles := int64(0)
walkFn := func(path string, file os.FileInfo, err error) error {
if path == basedir {
return nil
}
countFn(file)
match, err := fc.filter(file)
if err != nil {
acc.AddError(err)
return nil
}
if match {
numFiles++
}
if !recursive && file.IsDir() {
return filepath.SkipDir
}
return nil
}
return filepath.Walk(basedir, walkFn)

err := filepath.Walk(basedir, walkFn)
if err != nil {
acc.AddError(err)
return
}

acc.AddFields("filecount",
map[string]interface{}{
"count": numFiles,
},
map[string]string{
"directory": basedir,
},
)
}

func (fc *FileCount) initFileFilters() {
Expand Down Expand Up @@ -168,37 +201,53 @@ func (fc *FileCount) filter(file os.FileInfo) (bool, error) {
}

func (fc *FileCount) Gather(acc telegraf.Accumulator) error {
numFiles := int64(0)
countFn := func(f os.FileInfo) {
match, err := fc.filter(f)
if err != nil {
acc.AddError(err)
return
}
if !match {
return
}
numFiles++
}
err := count(fc.Directory, fc.Recursive, countFn)
globDirs := fc.getDirs()
dirs, err := getCompiledDirs(globDirs)
if err != nil {
acc.AddError(err)
return err
}

acc.AddFields("filecount",
map[string]interface{}{
"count": numFiles,
},
map[string]string{
"directory": fc.Directory,
})
for _, dir := range dirs {
fc.count(acc, dir, fc.Recursive)
}

return nil
}

func (fc *FileCount) getDirs() []string {
dirs := make([]string, len(fc.Directories))
for i, dir := range fc.Directories {
dirs[i] = dir
}

if fc.Directory != "" {
dirs = append(dirs, fc.Directory)
}

return dirs
}

func getCompiledDirs(dirs []string) ([]string, error) {
compiledDirs := []string{}
for _, dir := range dirs {
g, err := globpath.Compile(dir)
if err != nil {
return nil, fmt.Errorf("could not compile glob %v: %v", dir, err)
}

for path, file := range g.Match() {
if file.IsDir() {
compiledDirs = append(compiledDirs, path)
}
}
}
return compiledDirs, nil
}

func NewFileCount() *FileCount {
return &FileCount{
Directory: "",
Directories: []string{},
Name: "*",
Recursive: true,
RegularOnly: true,
Expand Down
95 changes: 66 additions & 29 deletions plugins/inputs/filecount/filecount_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,69 +14,108 @@ import (
)

func TestNoFilters(t *testing.T) {
fc := getNoFilterFileCount()
matches := []string{"foo", "bar", "baz", "qux",
"subdir/", "subdir/quux", "subdir/quuz"}
require.True(t, fileCountEquals(fc, len(matches)))
fc := getNoFilterFileCount("*")
matches := []string{"foo", "bar", "baz", "qux", "subdir/", "subdir/quux", "subdir/quuz"}

acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)

require.True(t, assertFileCount(&acc, "testdata", len(matches)))
}

func TestNoFiltersOnChildDir(t *testing.T) {
fc := getNoFilterFileCount("testdata/*")
matches := []string{"subdir/quux", "subdir/quuz"}

acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)

require.True(t, assertFileCount(&acc, "testdata/subdir", len(matches)))
}

func TestNameFilter(t *testing.T) {
fc := getNoFilterFileCount()
fc := getNoFilterFileCount("testdata")
fc.Name = "ba*"
matches := []string{"bar", "baz"}
require.True(t, fileCountEquals(fc, len(matches)))

acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)

require.True(t, assertFileCount(&acc, "testdata", len(matches)))
}

func TestNonRecursive(t *testing.T) {
fc := getNoFilterFileCount()
fc := getNoFilterFileCount("testdata")
fc.Recursive = false
matches := []string{"foo", "bar", "baz", "qux", "subdir"}
require.True(t, fileCountEquals(fc, len(matches)))

acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)

require.True(t, assertFileCount(&acc, "testdata", len(matches)))
}

func TestRegularOnlyFilter(t *testing.T) {
fc := getNoFilterFileCount()
fc := getNoFilterFileCount("testdata")
fc.RegularOnly = true
matches := []string{
"foo", "bar", "baz", "qux", "subdir/quux", "subdir/quuz",
}
require.True(t, fileCountEquals(fc, len(matches)))

acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)

require.True(t, assertFileCount(&acc, "testdata", len(matches)))
}

func TestSizeFilter(t *testing.T) {
fc := getNoFilterFileCount()
fc := getNoFilterFileCount("testdata")
fc.Size = -100
matches := []string{"foo", "bar", "baz",
"subdir/quux", "subdir/quuz"}
require.True(t, fileCountEquals(fc, len(matches)))
matches := []string{"foo", "bar", "baz", "subdir/quux", "subdir/quuz"}

acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)

require.True(t, assertFileCount(&acc, "testdata", len(matches)))

fc.Size = 100
matches = []string{"qux"}
require.True(t, fileCountEquals(fc, len(matches)))

acc = testutil.Accumulator{}
acc.GatherError(fc.Gather)

require.True(t, assertFileCount(&acc, "testdata", len(matches)))
}

func TestMTimeFilter(t *testing.T) {
oldFile := filepath.Join(getTestdataDir(), "baz")
oldFile := filepath.Join(getTestdataDir("testdata"), "baz")
mtime := time.Date(1979, time.December, 14, 18, 25, 5, 0, time.UTC)
if err := os.Chtimes(oldFile, mtime, mtime); err != nil {
t.Skip("skipping mtime filter test.")
}
fileAge := time.Since(mtime) - (60 * time.Second)

fc := getNoFilterFileCount()
fc := getNoFilterFileCount("testdata")
fc.MTime = internal.Duration{Duration: -fileAge}
matches := []string{"foo", "bar", "qux",
"subdir/", "subdir/quux", "subdir/quuz"}
require.True(t, fileCountEquals(fc, len(matches)))
matches := []string{"foo", "bar", "qux", "subdir/", "subdir/quux", "subdir/quuz"}

acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)

require.True(t, assertFileCount(&acc, "testdata", len(matches)))

fc.MTime = internal.Duration{Duration: fileAge}
matches = []string{"baz"}
require.True(t, fileCountEquals(fc, len(matches)))

acc = testutil.Accumulator{}
acc.GatherError(fc.Gather)

require.True(t, assertFileCount(&acc, "testdata", len(matches)))
}

func getNoFilterFileCount() FileCount {
func getNoFilterFileCount(dir string) FileCount {
return FileCount{
Directory: getTestdataDir(),
Directories: []string{getTestdataDir(dir)},
Name: "*",
Recursive: true,
RegularOnly: false,
Expand All @@ -86,14 +125,12 @@ func getNoFilterFileCount() FileCount {
}
}

func getTestdataDir() string {
func getTestdataDir(dir string) string {
_, filename, _, _ := runtime.Caller(1)
return strings.Replace(filename, "filecount_test.go", "testdata/", 1)
return strings.Replace(filename, "filecount_test.go", dir, 1)
}

func fileCountEquals(fc FileCount, expectedCount int) bool {
tags := map[string]string{"directory": getTestdataDir()}
acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)
func assertFileCount(acc *testutil.Accumulator, expectedDir string, expectedCount int) bool {
tags := map[string]string{"directory": getTestdataDir(expectedDir)}
return acc.HasPoint("filecount", tags, "count", int64(expectedCount))
}

0 comments on commit 030f944

Please sign in to comment.