Discard predicates for elements with direct location links in dbscheme

Author: d10c

python/ql/lib/python.qll

@@ -37,6 +37,7 @@ import semmle.python.pointsto.CallGraph
 import semmle.python.objects.ObjectAPI
 import semmle.python.Unit
 import site
+private import semmle.python.Overlay
 // Removing this import perturbs the compilation process enough that the points-to analysis gets
 // compiled -- and cached -- differently depending on whether the data flow library is imported. By
 // importing it privately here, we ensure that the points-to analysis is compiled the same way.

python/ql/lib/semmle/python/Overlay.qll

@@ -0,0 +1,132 @@
+/**
+ * Defines entity discard predicates for Python overlay analysis.
+ */
+overlay[local?]
+module;
+
+import python
+
+/**
+ * A local predicate that always holds for the overlay variant and
+ * never holds for the base variant. This is used to define local
+ * predicates that behave differently for the base and overlay variant.
+ */
+overlay[local]
+predicate isOverlay() { databaseMetadata("isOverlay", "true") }
+
+/** Gets the raw file path for a locatable Python AST node. */
+overlay[local]
+private string getRawPathForLocatable(@py_location_parent el) {
+  exists(@location loc | py_locations(loc, el) | result = getRawPathForLocation(loc))
+}
+
+/** Gets the raw path for a location. */
+overlay[local]
+private string getRawPathForLocation(@location loc) {
+  exists(@file file | locations_default(loc, file, _, _, _, _) | files(file, result))
+  or
+  exists(@py_Module mod | locations_ast(loc, mod, _, _, _, _) | result = getRawPathForModule(mod))
+}
+
+overlay[local]
+private string getRawPathForModule(@py_Module mod) {
+  exists(@container fileOrFolder | py_module_path(mod, fileOrFolder) |
+    result = getRawPathForContainer(fileOrFolder)
+  )
+}
+
+overlay[local]
+private string getRawPathForScope(@py_scope scope) {
+  exists(@location loc | py_scope_location(loc, scope) | result = getRawPathForLocation(loc))
+  or
+  result = getRawPathForModule(scope.(@py_Module))
+}
+
+overlay[local]
+private string getRawPathForContainer(@container fileOrFolder) {
+  files(fileOrFolder, result) or folders(fileOrFolder, result)
+}
+
+/**
+ * Discard all locatable AST nodes (`@py_location_parent`) in modified files
+ * since they use *-ids and hence cannot be referenced across TRAP files.
+ */
+overlay[discard_entity]
+private predicate discardLocatable(@py_location_parent el) {
+  overlayChangedFiles(getRawPathForLocatable(el)) and
+  not isOverlay()
+}
+
+overlay[local]
+private predicate scopeExistsInBase(@py_scope scope) { not isOverlay() and exists(scope) }
+
+overlay[local]
+private predicate scopeExistsInOverlay(@py_scope scope) { isOverlay() and exists(scope) }
+
+/**
+ * Discard scopes (classes, functions, modules) that were deleted in the overlay.
+ */
+overlay[discard_entity]
+private predicate discardScope(@py_scope scope) {
+  overlayChangedFiles(getRawPathForScope(scope)) and
+  scopeExistsInBase(scope) and
+  not scopeExistsInOverlay(scope)
+}
+
+overlay[local]
+private predicate containerExistsInBase(@container fileOrFolder) {
+  not isOverlay() and exists(fileOrFolder)
+}
+
+overlay[local]
+private predicate containerExistsInOverlay(@container fileOrFolder) {
+  isOverlay() and exists(fileOrFolder)
+}
+
+/**
+ * Discard files and folders that were deleted in the overlay.
+ */
+overlay[discard_entity]
+private predicate discardContainer(@container fileOrFolder) {
+  overlayChangedFiles(getRawPathForContainer(fileOrFolder)) and
+  containerExistsInBase(fileOrFolder) and
+  not containerExistsInOverlay(fileOrFolder)
+}
+
+overlay[local]
+private predicate baseXmlLocatable(@xmllocatable l) {
+  not isOverlay() and not files(l, _) and not xmlNs(l, _, _, _)
+}
+
+overlay[local]
+private predicate overlayHasXmlLocatable() {
+  isOverlay() and
+  exists(@xmllocatable l | not files(l, _) and not xmlNs(l, _, _, _))
+}
+
+overlay[discard_entity]
+private predicate discardBaseXmlLocatable(@xmllocatable el) {
+  // The XML extractor is currently not incremental, so if
+  // the overlay contains any XML locatables, the overlay should
+  // contain a full extraction and all XML locatables from base
+  // should be discarded.
+  baseXmlLocatable(el) and overlayHasXmlLocatable()
+}
+
+overlay[local]
+private predicate baseYamlLocatable(@yaml_locatable l) { not isOverlay() and exists(l) }
+
+overlay[local]
+private predicate overlayHasYamlLocatable() {
+  isOverlay() and
+  exists(@yaml_locatable l)
+}
+
+overlay[discard_entity]
+private predicate discardBaseYamlLocatable(@yaml_locatable el) {
+  // The Yaml extractor is currently not incremental, so if
+  // the overlay contains any Yaml locatables, the overlay should
+  // contain a full extraction and all Yaml locatables from base
+  // should be discarded.
+  baseYamlLocatable(el) and overlayHasYamlLocatable()
+}