carbonserver proposed fix

carbonserver.go

func (listener *CarbonserverListener) updateAccessedMetrics(details map[string]*pb.MetricDetails) map[string]struct{} {
	fidx := listener.CurrentFileIndex()
	accessedMetrics := make(map[string]struct{})
	if fidx != nil {
		fidx.Lock()
		defer fidx.Unlock()
		for m := range fidx.accessedMetrics {
			accessedMetrics[m] = struct{}{}
		}

		for m := range fidx.accessedMetrics {
			if d, ok := details[m]; ok {
				d.RdTime = fidx.details[m].RdTime
			}
		}
	}
	return accessedMetrics
}

func (listener *CarbonserverListener) updateFileList(dir string) {
	logger := listener.logger.With(zap.String("handler", "fileListUpdated"))
	defer func() {
		if r := recover(); r != nil {
			logger.Error("panic encountered",
				zap.Stack("stack"),
				zap.Any("error", r),
			)
		}
	}()
	t0 := time.Now()

	var files []string
	details := make(map[string]*pb.MetricDetails)

	metricsKnown := uint64(0)
	err := filepath.Walk(dir, func(p string, info os.FileInfo, err error) error {
		if err != nil {
			logger.Info("error processing", zap.String("path", p), zap.Error(err))
			return nil
		}
		i := stat.GetStat(info)

		hasSuffix := strings.HasSuffix(info.Name(), ".wsp")
		if info.IsDir() || hasSuffix {
			trimmedName := strings.TrimPrefix(p, listener.whisperData)
			files = append(files, trimmedName)
			if hasSuffix {
				metricsKnown++
				trimmedName = strings.Replace(trimmedName[1:len(trimmedName)-4], "/", ".", -1)
				details[trimmedName] = &pb.MetricDetails{
					Size_:   i.RealSize,
					ModTime: i.MTime,
					ATime:   i.ATime,
				}
			}
		}

		return nil
	})
	if err != nil {
		logger.Error("error getting file list",
			zap.Error(err),
		)
	}

	var stat syscall.Statfs_t
	err = syscall.Statfs(dir, &stat)
	if err != nil {
		logger.Info("error getting FS Stats",
			zap.String("dir", dir),
			zap.Error(err),
		)
		return
	}

	freeSpace := stat.Bavail * uint64(stat.Bsize)
	totalSpace := stat.Blocks * uint64(stat.Bsize)

	fileScanRuntime := time.Since(t0)
	atomic.StoreUint64(&listener.metrics.MetricsKnown, metricsKnown)
	atomic.AddUint64(&listener.metrics.FileScanTimeNS, uint64(fileScanRuntime.Nanoseconds()))

	t0 = time.Now()
	idx := trigram.NewIndex(files)

	indexingRuntime := time.Since(t0)
	atomic.AddUint64(&listener.metrics.IndexBuildTimeNS, uint64(indexingRuntime.Nanoseconds()))
	indexSize := len(idx)

	pruned := idx.Prune(0.95)

	tl := time.Now()
	accessedMetrics := listener.updateAccessedMetrics(details)
	rdTimeUpdateRuntime := time.Since(tl)

	listener.UpdateFileIndex(&fileIndex{
		idx:             idx,
		files:           files,
		details:         details,
		freeSpace:       freeSpace,
		totalSpace:      totalSpace,
		accessedMetrics: accessedMetrics,
	})

	logger.Info("file list updated",
		zap.Duration("file_scan_runtime", fileScanRuntime),
		zap.Duration("indexing_runtime", indexingRuntime),
		zap.Duration("rdtime_update_runtime", rdTimeUpdateRuntime),
		zap.Duration("total_runtime", time.Since(t0)),
		zap.Int("files", len(files)),
		zap.Int("index_size", indexSize),
		zap.Int("pruned_trigrams", pruned),
	)
}