Skip to content

Commit

Permalink
DEPTRACKER-25: added option for soft record cache during scanning to …
Browse files Browse the repository at this point in the history 
…reduce memory footprint
  • Loading branch information
@jjannek
jjannek committed Oct 25, 2024
1 parent 3234575 commit 8e6b4b7
Show file tree
Hide file tree
Showing 5 changed files with 85 additions and 20 deletions.
4 changes: 3 additions & 1 deletion forms/system-config/content-dependency-tracker.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@
</tab>
<tab id="advanced">
<fieldset id="default">
<field name="delta_scanning_batch_size" control="spinner" />
<field name="delta_scanning_batch_size" control="spinner" />
<field name="enable_soft_record_cache_full" control="yesnoswitch" />
<field name="enable_soft_record_cache_delta" control="yesnoswitch" />
</fieldset>
</tab>
</form>
8 changes: 7 additions & 1 deletion i18n/system-config/content-dependency-tracker.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,4 +27,10 @@ field.single_record_scanning.title=Single Record Scanning
field.single_record_scanning.help=Whether a single record should be scanned for dependencies on insert, update or delete

field.delta_scanning_batch_size.title=Delta Scanning Batch Size
field.delta_scanning_batch_size.help=The number of records to be scanned in a single batch when performing delta scanning (frequently scanning records marked to be scanned), leave empty to scan all flagged records in each run
field.delta_scanning_batch_size.help=The number of records to be scanned in a single batch when performing delta scanning (frequently scanning records marked to be scanned), leave empty to scan all flagged records in each run

field.enable_soft_record_cache_full.title=Enable Soft Record Cache (Full Scan)
field.enable_soft_record_cache_full.help=By default all tracked records are cached in memory in each full scan. This can be disabled to reduce memory usage, but will increase the number of database queries. In case of performance issues, try enabling or disabling this option.

field.enable_soft_record_cache_delta.title=Enable Soft Record Cache (Delta Scan)
field.enable_soft_record_cache_delta.help=By default all tracked records are cached in memory in each delta scan. This can be disabled to reduce memory usage, but will increase the number of database queries. In case of performance issues, try enabling or disabling this option.
2 changes: 1 addition & 1 deletion preside-objects/tracked_content_record.cfc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ component {
property name="label" type="string" dbtype="varchar" maxlength=500 required=true renderer="trackedContentRecordLabelRenderer";
property name="id" type="numeric" dbtype="bigint" required=true generator="increment";
property name="object_name" type="string" dbtype="varchar" maxlength=50 required=true renderer="objectName" uniqueIndexes="objectNameAndRecordId|1" enum="dependencyTrackerObjectNames";
property name="record_id" type="string" dbtype="varchar" maxlength=35 required=true uniqueIndexes="objectNameAndRecordId|2";
property name="record_id" type="string" dbtype="varchar" maxlength=35 required=true uniqueIndexes="objectNameAndRecordId|2" indexes="recordId";
property name="orphaned" type="boolean" dbtype="bit" default="false" required=true indexes="orphaned";
property name="hidden" type="boolean" dbtype="bit" default="false" required=true indexes="hidden";
property name="requires_scanning" type="boolean" dbtype="bit" default="false" required=true indexes="requires_scanning";
Expand Down
8 changes: 8 additions & 0 deletions services/ContentDependencyTrackerConfigurationService.cfc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,14 @@ component {
return _isBooleanSystemSettingEnabled( setting="hide_all_irrelevant_records" );
}

public boolean function isSoftRecordCacheInFullScanEnabled() {
return _isBooleanSystemSettingEnabled( setting="enable_soft_record_cache_full" );
}

public boolean function isSoftRecordCacheInDeltaScanEnabled() {
return _isBooleanSystemSettingEnabled( setting="enable_soft_record_cache_delta" );
}

public struct function getLinkToTrackerEventConfig() {
var settings = _getSettings();
return settings.linkToTrackerEvents ?: {};
Expand Down
83 changes: 66 additions & 17 deletions services/ContentDependencyTrackerService.cfc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,9 @@ component {
_setFullProcessing( arguments.full );
_setProcessId( createUUID() );
_setProcessTimestamp( now() );
_storeSoftRecordCacheEnabled( arguments.full );

logger.info( "Now scanning content records to track dependencies (Process ID: #_getProcessId()#, Full: #arguments.full#, FK Scanning enabled: #isForeignKeyScanningEnabled#, Soft Reference Scanning enabled: #isSoftReferenceScanningEnabled#)..." );
logger.info( "Now scanning content records to track dependencies (Process ID: #_getProcessId()#, Full: #arguments.full#, FK Scanning enabled: #isForeignKeyScanningEnabled#, Soft Reference Scanning enabled: #isSoftReferenceScanningEnabled#, Soft Record Cache enabled: #_softCachingOnly()#)..." );

var contentRecordIdMap = {};
var objectNames = _getConfiguration().getAllTrackableObjects();
Expand Down Expand Up @@ -291,7 +292,6 @@ component {
filter = "orphaned = :orphaned and exists (select 1 from pobj_tracked_content_record_dependency d where d.content_record = tracked_content_record.id or d.dependent_content_record = tracked_content_record.id)"
, filterParams = { orphaned=true }
, recordCountOnly = true
, useCache = false
);
if ( broken > 0 ) {
logger.info( "Found [#broken#] orphaned content record(s) that other content records depend on (Broken dependencies). Those have not been deleted." );
Expand Down Expand Up @@ -323,7 +323,6 @@ component {
var records = _getContentRecordDao().selectData(
filter = { object_name=arguments.objectName, record_id=arguments.recordId }
, selectFields = [ "id" ]
, useCache = false
);

for ( var record in records ) {
Expand All @@ -337,7 +336,6 @@ component {
var records = _getContentRecordDao().selectData(
filter = { object_name=arguments.objectName, record_id=arguments.recordId }
, selectFields = [ "id", "label", "depends_on_count", "dependent_by_count" ]
, useCache = false
);

for ( var record in records ) {
Expand Down Expand Up @@ -510,7 +508,7 @@ component {
filter[ "record_id" ] = arguments.recordIds;
}

var q = _getContentRecordDao().selectData( selectFields=[ "id", "label", "orphaned" ], filter=filter, useCache=false );
var q = _getContentRecordDao().selectData( selectFields=[ "id", "label", "orphaned" ], filter=filter );
var result = { labels={}, orphaned={} };

loop query="q" {
Expand Down Expand Up @@ -710,19 +708,36 @@ component {
private any function _getTrackedContentRecordIds() {
return _simpleLocalCache( "getTrackedContentRecordIds", function() {

var records = _getContentRecordDao().selectData( selectFields=[ "record_id" ], distinct=true, useCache=false );
var result = createObject( "java", "java.util.HashSet" ).init();

if ( records.recordCount ) {
result.addAll( queryColumnData( records, "record_id" ) );
if ( _cacheAllRecords() ) {
var records = _getContentRecordDao().selectData( selectFields=[ "record_id" ], distinct=true );

if ( records.recordCount ) {
result.addAll( queryColumnData( records, "record_id" ) );
}
}

return result;
} );
}

private boolean function _isTrackedContentRecordId( required any recordId ) {
return _getTrackedContentRecordIds().contains( arguments.recordId );

var foundInCache = _getTrackedContentRecordIds().contains( arguments.recordId );

if ( _cacheAllRecords() || foundInCache ) {
return foundInCache;
}

// soft caching only and not found - check the DB
var foundInDb = _getContentRecordDao().dataExists( filter={ record_id=arguments.recordId } );

if ( foundInDb ) {
_addTrackedContentRecordId( arguments.recordId );
}

return foundInDb;
}

private void function _addTrackedContentRecordId( required string recordId ) {
Expand All @@ -733,11 +748,14 @@ component {
private struct function _getMappedContentRecordIds() {
return _simpleLocalCache( "getMappedContentRecordIds", function() {

var records = _getContentRecordDao().selectData( selectFields=[ "record_id", "object_name", "id" ], useCache=false );
var result = {};

loop query="records" {
result[ "#records.object_name#_#records.record_id#" ] = records.id;
if ( _cacheAllRecords() ) {
var records = _getContentRecordDao().selectData( selectFields=[ "record_id", "object_name", "id" ] );

loop query="records" {
result[ "#records.object_name#_#records.record_id#" ] = records.id;
}
}

return result;
Expand All @@ -750,6 +768,7 @@ component {
}

private numeric function _mapContentRecordId( required string objectName, required string recordId ) {
// assumes that _mappedContentRecordExists() has been called before
var mappings = _getMappedContentRecordIds();
return mappings[ arguments.objectName & "_" & arguments.recordId ] ?: 0;
}
Expand Down Expand Up @@ -788,20 +807,52 @@ component {
}

private boolean function _mappedContentRecordExists( required string objectName, required string recordId ) {
return structKeyExists( _getMappedContentRecordIds(), arguments.objectName & "_" & arguments.recordId );

var foundInCache = structKeyExists( _getMappedContentRecordIds(), arguments.objectName & "_" & arguments.recordId );

if ( _cacheAllRecords() || foundInCache ) {
return foundInCache;
}

// soft caching only and not found in cache - check the DB additionally
var record = _getContentRecordDao().selectData(
filter = { object_name=arguments.objectName, record_id=arguments.recordId }
, selectFields = [ "id" ]
);

if ( record.recordCount ) {
_addMappedContentRecordId( objectName=arguments.objectName, recordId=arguments.recordId, mappedId=record.id );
return true;
}

return false;
}

private void function _cacheContentRecordData() {
_clearCachedContentRecordData();
_getTrackedContentRecordIds();
_getMappedContentRecordIds();
if ( _cacheAllRecords() ) {
_getTrackedContentRecordIds();
_getMappedContentRecordIds();
}
}

private void function _clearCachedContentRecordData() {
structDelete( _getLocalCache(), "getTrackedContentRecordIds" );
structDelete( _getLocalCache(), "getMappedContentRecordIds" );
}

private boolean function _softCachingOnly() {
return _softRecordCacheEnabled ?: false;
}

private boolean function _cacheAllRecords() {
return !_softCachingOnly();
}

private void function _storeSoftRecordCacheEnabled( required boolean full ) {
_softRecordCacheEnabled = arguments.full ? _getConfiguration().isSoftRecordCacheInFullScanEnabled() : _getConfiguration().isSoftRecordCacheInDeltaScanEnabled();
}

private array function _findUuids( required string content ) {
// this will find plain UUIDs, but also those that have been url encoded one or more times (this is the case in rich editor content with nested widgets)
var plainOrUrlEncodedUuidRegexPattern = "[0-9a-fA-F]{8}(-|%(25)*2D)[0-9a-fA-F]{4}(-|%(25)*2D)[0-9a-fA-F]{4}(-|%(25)*2D)[0-9a-fA-F]{16}";
Expand Down Expand Up @@ -829,7 +880,6 @@ component {
, selectFields = [ "object_name", "record_id" ]
, orderby = "datemodified"
, maxRows = arguments.batchSize
, useCache = false
);

var result = {};
Expand All @@ -848,7 +898,6 @@ component {
return _getContentRecordDao().selectData(
filter = { requires_scanning=true }
, recordCountOnly = true
, useCache = false
);
}

Expand Down

0 comments on commit 8e6b4b7

Please sign in to comment.