pmdaopenmetrics metric removal qa bug fix

addresses multiple qa test failures. all qa tests
now pass, and updated test #1976 checks for metric
removal from a modified source that still persists
in the config directory.

Author: lmchilton

qa/1976

@@ -78,6 +78,17 @@ $sudo touch -t 197001010000 $urlfile
 pminfo openmetrics.simple_metric
 echo
 
+echo "-- metric removal by modifying source file, source persists -- "
+path=$tmp'.simple_metric.txt'
+$sudo cp $here/openmetrics/samples/simple_metric.txt $tmp.saved_metric2
+$sudo cp $here/openmetrics/samples/simple_metric.txt $tmp2.saved_metric2
+$sudo sed -i -e "/metric2/d" $path
+$sudo sed -i -e "/metric2/d"  $here/openmetrics/samples/simple_metric.txt
+pminfo openmetrics.simple_metric
+$sudo mv $tmp.saved_metric2 $path
+$sudo mv $tmp2.saved_metric2 $here/openmetrics/samples/simple_metric.txt
+
+
 _pmdaopenmetrics_remove >/dev/null 2>&1
 
 # success, all done

qa/1976.out

@@ -15,3 +15,5 @@ Error: openmetrics.simple_metric: Unknown metric name
 openmetrics.simple_metric.metric2
 openmetrics.simple_metric.metric1
 
+-- metric removal by modifying source file, source persists -- 
+openmetrics.simple_metric.metric1

src/pmdas/openmetrics/pmdaopenmetrics.python

@@ -30,6 +30,7 @@ import argparse
 import threading
 import subprocess
 import sys
+from urllib.parse import urlparse
 from ctypes import c_int
 from socket import gethostname
 from stat import ST_MODE, S_IXUSR
@@ -908,7 +909,6 @@ class Source(object):
         # clear cached values from all my metrics
         for _, m in self.metrics_by_name.items():
             m.clear_values()
-        # TODO: ditch metrics no longer found in document
 
         # bump the fetch call counter for this source, and for the total (cluster 0)
         self.pmda.stats_fetch_calls[self.cluster] += 1
@@ -948,6 +948,24 @@ class Source(object):
                     # NB: the requests package automatically enables http keep-alive and compression
                     self.document = r.text
 
+            # ditch metrics no longer found in document
+            for name, metric in self.pmda.all_metrics.items():
+                split_metric = name.split(".")
+                stripped_metric = name.replace("openmetrics.%s." % self.name, "")
+                self.pmda.debug("stripped metric --> %s" % stripped_metric) if self.pmda.dbg else None
+                if stripped_metric == "metric2":
+                    self.pmda.debug("self.document: %s" % self.document) if self.pmda.dbg else None
+                if split_metric[1] == self.name and stripped_metric not in self.document:
+                    self.pmda.debug("true") if self.pmda.dbg else None
+                    try:
+                        self.pmda.remove_metric(name, metric)
+                        self.pmda.removed_metrics[name] = metric
+                        self.pmda.set_need_refresh()
+                        self.pmda.debug("removed metric from existing source: %s" % name) if self.pmda.dbg else None
+                        del self.pmda.all_metrics[name]
+                    except Exception as e:
+                        self.pmda.debug("can't remove metric: %s, see error: %s" % (name, e)) if self.pmda.dbg else None
+
             # update fetch time counter stats, in ms
             incr = int(1000 * (time.time() - fetch_time))
             self.pmda.stats_fetch_time[self.cluster] += incr
@@ -1040,6 +1058,10 @@ class OpenMetricsPMDA(PMDA):
         self.all_metrics = {}
         # keep track of removed metrics, in case of re-addition
         self.removed_metrics = {}
+        # keep track of control initialization
+        self.controls = {0:0}
+        # keep track of modified sources, in need of extra refresh
+        self.modified_files = []
 
         # compiled regex cache
         self.regex_cache = {}
@@ -1147,12 +1169,40 @@ class OpenMetricsPMDA(PMDA):
                     if mtime is None or m > mtime:
                         mtime = m
                     ret.append(fname)
+                fname = os.path.join(path, f)
+                with open(fname, 'r') as name:
+                    f_path = name.readline().strip()
+                    if f_path.startswith("file:///"):
+                        parsed = urlparse(f_path)
+                        m = os.path.getmtime(parsed.path)
+                        if mtime is None or m > mtime:
+                            mtime = m
+                            self.modified_files.append(fname)
             for d in subdirs:
                 m, _ = self.traverse(os.path.join(path, d), mtime)
                 if mtime is None or m > mtime:
                     mtime = m
         return mtime, ret
 
+    def initialize_controls(self, cluster):
+        # initialize statistics
+        self.stats_fetch_calls[cluster] = 0
+        self.stats_fetch_time[cluster] = 0
+        self.stats_parse_time[cluster] = 0
+        self.stats_status[cluster] = "unknown"
+        self.stats_status_code[cluster] = 0
+
+        self.controls[cluster] = 1
+
+    def delete_controls(self, cluster):
+        del self.stats_fetch_calls[cluster]
+        del self.stats_fetch_time[cluster]
+        del self.stats_parse_time[cluster]
+        del self.stats_status[cluster]
+        del self.stats_status_code[cluster]
+
+        self.controls[cluster] = 0
+
     def rescan_confdir(self):
         '''Scan the configuration directories for any new .url files
         or scripts.  Ensure there is a Source registered in the
@@ -1163,6 +1213,10 @@ class OpenMetricsPMDA(PMDA):
         invoked frequently by src/python/pmda.c.
         '''
 
+        # re-initialize variables to keep track of modified files before new traversal
+        self.modified_files = []
+        m_files = []
+
         traverse_time = time.time()
         dir_mtime, conf_filelist = self.traverse(self.config_dir, self.config_dir_mtime)
         traverse_time = time.time() - traverse_time
@@ -1183,6 +1237,11 @@ class OpenMetricsPMDA(PMDA):
             name = file_split[0].replace(self.config_dir + "/", "").replace("/", ".")
             nicknames.append(name)
 
+        for file in self.modified_files:
+            file_split = os.path.splitext(file)
+            name = file_split[0].replace(self.config_dir + "/", "").replace("/", ".")
+            m_files.append(name)
+
         # check if config change adds a previously removed source
         for key in self.removed_metrics:
             split_name = key.split(".")
@@ -1197,6 +1256,9 @@ class OpenMetricsPMDA(PMDA):
             try:
                 remove_name = key
                 remove_obj = value
+                cluster = self.cluster_table.intern_lookup_value(split_name[1])
+                if self.controls[cluster] == 1:
+                    self.delete_controls(cluster)
                 self.remove_metric(remove_name, remove_obj)
                 self.removed_metrics[remove_name] = remove_obj
                 self.debug("removed metric name: %s" % remove_name) if self.dbg else None
@@ -1236,13 +1298,16 @@ class OpenMetricsPMDA(PMDA):
                 # this source is already known
                 self.assert_source_invariants(name=name)
                 s = self.source_by_name[name]
+                self.debug("m files : %s" % m_files) if self.dbg else None
                 cluster_for_refresh = []
                 cluster_for_refresh_names = []
-                if name in self.re_add_list:
+                if name in self.re_add_list or name in m_files:
                     for key,value in self.source_by_cluster.items():
                         if value == s:
                             cluster_for_refresh.append(key)
                             cluster_for_refresh_names.append(name)
+                            if self.controls[key] == 0:
+                                self.initialize_controls(key)
                     self.debug("refreshing cluster list: %s" % cluster_for_refresh_names) if self.dbg else None
                     self.refresh_some_clusters_for_fetch(cluster_for_refresh)
             else:
@@ -1253,12 +1318,7 @@ class OpenMetricsPMDA(PMDA):
                     self.source_by_name[source.name] = source
                     self.source_by_cluster[source.cluster] = source
 
-                    # initialize statistics
-                    self.stats_fetch_calls[cluster] = 0
-                    self.stats_fetch_time[cluster] = 0
-                    self.stats_parse_time[cluster] = 0
-                    self.stats_status[cluster] = "unknown"
-                    self.stats_status_code[cluster] = 0
+                    self.initialize_controls(cluster)
 
                     save_cluster_table = True
                     self.log("Found source %s cluster %d" % (name, cluster))