Branch data Line data Source code
1 : : /***************************************************************************
2 : : qgsalgorithmdetectdatasetchanges.cpp
3 : : -----------------------------------------
4 : : begin : December 2019
5 : : copyright : (C) 2019 by Nyall Dawson
6 : : email : nyall dot dawson at gmail dot com
7 : : ***************************************************************************/
8 : :
9 : : /***************************************************************************
10 : : * *
11 : : * This program is free software; you can redistribute it and/or modify *
12 : : * it under the terms of the GNU General Public License as published by *
13 : : * the Free Software Foundation; either version 2 of the License, or *
14 : : * (at your option) any later version. *
15 : : * *
16 : : ***************************************************************************/
17 : :
18 : : #include "qgsalgorithmdetectdatasetchanges.h"
19 : : #include "qgsvectorlayer.h"
20 : : #include "qgsgeometryengine.h"
21 : :
22 : : ///@cond PRIVATE
23 : :
24 : 0 : QString QgsDetectVectorChangesAlgorithm::name() const
25 : : {
26 : 0 : return QStringLiteral( "detectvectorchanges" );
27 : : }
28 : :
29 : 0 : QString QgsDetectVectorChangesAlgorithm::displayName() const
30 : : {
31 : 0 : return QObject::tr( "Detect dataset changes" );
32 : : }
33 : :
34 : 0 : QStringList QgsDetectVectorChangesAlgorithm::tags() const
35 : : {
36 : 0 : return QObject::tr( "added,dropped,new,deleted,features,geometries,difference,delta,revised,original,version" ).split( ',' );
37 : 0 : }
38 : :
39 : 0 : QString QgsDetectVectorChangesAlgorithm::group() const
40 : : {
41 : 0 : return QObject::tr( "Vector general" );
42 : : }
43 : :
44 : 0 : QString QgsDetectVectorChangesAlgorithm::groupId() const
45 : : {
46 : 0 : return QStringLiteral( "vectorgeneral" );
47 : : }
48 : :
49 : 0 : void QgsDetectVectorChangesAlgorithm::initAlgorithm( const QVariantMap & )
50 : : {
51 : 0 : addParameter( new QgsProcessingParameterFeatureSource( QStringLiteral( "ORIGINAL" ), QObject::tr( "Original layer" ) ) );
52 : 0 : addParameter( new QgsProcessingParameterFeatureSource( QStringLiteral( "REVISED" ), QObject::tr( "Revised layer" ) ) );
53 : :
54 : 0 : std::unique_ptr< QgsProcessingParameterField > compareAttributesParam = std::make_unique< QgsProcessingParameterField >( QStringLiteral( "COMPARE_ATTRIBUTES" ),
55 : 0 : QObject::tr( "Attributes to consider for match (or none to compare geometry only)" ), QVariant(),
56 : 0 : QStringLiteral( "ORIGINAL" ), QgsProcessingParameterField::Any, true, true );
57 : 0 : compareAttributesParam->setDefaultToAllFields( true );
58 : 0 : addParameter( compareAttributesParam.release() );
59 : :
60 : 0 : std::unique_ptr< QgsProcessingParameterDefinition > matchTypeParam = std::make_unique< QgsProcessingParameterEnum >( QStringLiteral( "MATCH_TYPE" ),
61 : 0 : QObject::tr( "Geometry comparison behavior" ),
62 : 0 : QStringList() << QObject::tr( "Exact Match" )
63 : 0 : << QObject::tr( "Tolerant Match (Topological Equality)" ),
64 : 0 : false, 1 );
65 : 0 : matchTypeParam->setFlags( matchTypeParam->flags() | QgsProcessingParameterDefinition::FlagAdvanced );
66 : 0 : addParameter( matchTypeParam.release() );
67 : :
68 : 0 : addParameter( new QgsProcessingParameterFeatureSink( QStringLiteral( "UNCHANGED" ), QObject::tr( "Unchanged features" ), QgsProcessing::TypeVectorAnyGeometry, QVariant(), true, true ) );
69 : 0 : addParameter( new QgsProcessingParameterFeatureSink( QStringLiteral( "ADDED" ), QObject::tr( "Added features" ), QgsProcessing::TypeVectorAnyGeometry, QVariant(), true, true ) );
70 : 0 : addParameter( new QgsProcessingParameterFeatureSink( QStringLiteral( "DELETED" ), QObject::tr( "Deleted features" ), QgsProcessing::TypeVectorAnyGeometry, QVariant(), true, true ) );
71 : :
72 : 0 : addOutput( new QgsProcessingOutputNumber( QStringLiteral( "UNCHANGED_COUNT" ), QObject::tr( "Count of unchanged features" ) ) );
73 : 0 : addOutput( new QgsProcessingOutputNumber( QStringLiteral( "ADDED_COUNT" ), QObject::tr( "Count of features added in revised layer" ) ) );
74 : 0 : addOutput( new QgsProcessingOutputNumber( QStringLiteral( "DELETED_COUNT" ), QObject::tr( "Count of features deleted from original layer" ) ) );
75 : 0 : }
76 : :
77 : 0 : QString QgsDetectVectorChangesAlgorithm::shortHelpString() const
78 : : {
79 : 0 : return QObject::tr( "This algorithm compares two vector layers, and determines which features are unchanged, added or deleted between "
80 : : "the two. It is designed for comparing two different versions of the same dataset.\n\n"
81 : : "When comparing features, the original and revised feature geometries will be compared against each other. Depending "
82 : : "on the Geometry Comparison Behavior setting, the comparison will either be made using an exact comparison (where "
83 : : "geometries must be an exact match for each other, including the order and count of vertices) or a topological "
84 : : "comparison only (where are geometries area considered equal if all of their component edges overlap. E.g. "
85 : : "lines with the same vertex locations but opposite direction will be considered equal by this method). If the topological "
86 : : "comparison is selected then any z or m values present in the geometries will not be compared.\n\n"
87 : : "By default, the algorithm compares all attributes from the original and revised features. If the Attributes to Consider for Match "
88 : : "parameter is changed, then only the selected attributes will be compared (e.g. allowing users to ignore a timestamp or ID field "
89 : : "which is expected to change between the revisions).\n\n"
90 : : "If any features in the original or revised layers do not have an associated geometry, then care must be taken to ensure "
91 : : "that these features have a unique set of attributes selected for comparison. If this condition is not met, warnings will be "
92 : : "raised and the resultant outputs may be misleading.\n\n"
93 : : "The algorithm outputs three layers, one containing all features which are considered to be unchanged between the revisions, "
94 : : "one containing features deleted from the original layer which are not present in the revised layer, and one containing features "
95 : : "add to the revised layer which are not present in the original layer." );
96 : : }
97 : :
98 : 0 : QString QgsDetectVectorChangesAlgorithm::shortDescription() const
99 : : {
100 : 0 : return QObject::tr( "Calculates features which are unchanged, added or deleted between two dataset versions." );
101 : : }
102 : :
103 : 0 : QgsDetectVectorChangesAlgorithm *QgsDetectVectorChangesAlgorithm::createInstance() const
104 : : {
105 : 0 : return new QgsDetectVectorChangesAlgorithm();
106 : : }
107 : :
108 : 0 : bool QgsDetectVectorChangesAlgorithm::prepareAlgorithm( const QVariantMap ¶meters, QgsProcessingContext &context, QgsProcessingFeedback *feedback )
109 : : {
110 : 0 : mOriginal.reset( parameterAsSource( parameters, QStringLiteral( "ORIGINAL" ), context ) );
111 : 0 : if ( !mOriginal )
112 : 0 : throw QgsProcessingException( invalidSourceError( parameters, QStringLiteral( "ORIGINAL" ) ) );
113 : :
114 : 0 : mRevised.reset( parameterAsSource( parameters, QStringLiteral( "REVISED" ), context ) );
115 : 0 : if ( !mRevised )
116 : 0 : throw QgsProcessingException( invalidSourceError( parameters, QStringLiteral( "REVISED" ) ) );
117 : :
118 : 0 : mMatchType = static_cast< GeometryMatchType >( parameterAsEnum( parameters, QStringLiteral( "MATCH_TYPE" ), context ) );
119 : :
120 : 0 : switch ( mMatchType )
121 : : {
122 : : case Exact:
123 : 0 : if ( mOriginal->wkbType() != mRevised->wkbType() )
124 : 0 : throw QgsProcessingException( QObject::tr( "Geometry type of revised layer (%1) does not match the original layer (%2). Consider using the \"Tolerant Match\" option instead." ).arg( QgsWkbTypes::displayString( mRevised->wkbType() ),
125 : 0 : QgsWkbTypes::displayString( mOriginal->wkbType() ) ) );
126 : 0 : break;
127 : :
128 : : case Topological:
129 : 0 : if ( QgsWkbTypes::geometryType( mOriginal->wkbType() ) != QgsWkbTypes::geometryType( mRevised->wkbType() ) )
130 : 0 : throw QgsProcessingException( QObject::tr( "Geometry type of revised layer (%1) does not match the original layer (%2)" ).arg( QgsWkbTypes::geometryDisplayString( QgsWkbTypes::geometryType( mRevised->wkbType() ) ),
131 : 0 : QgsWkbTypes::geometryDisplayString( QgsWkbTypes::geometryType( mOriginal->wkbType() ) ) ) );
132 : 0 : break;
133 : :
134 : : }
135 : :
136 : 0 : if ( mOriginal->sourceCrs() != mRevised->sourceCrs() )
137 : 0 : feedback->reportError( QObject::tr( "CRS for revised layer (%1) does not match the original layer (%2) - reprojection accuracy may affect geometry matching" ).arg( mOriginal->sourceCrs().userFriendlyIdentifier(),
138 : 0 : mRevised->sourceCrs().userFriendlyIdentifier() ), false );
139 : :
140 : 0 : mFieldsToCompare = parameterAsFields( parameters, QStringLiteral( "COMPARE_ATTRIBUTES" ), context );
141 : 0 : mOriginalFieldsToCompareIndices.reserve( mFieldsToCompare.size() );
142 : 0 : mRevisedFieldsToCompareIndices.reserve( mFieldsToCompare.size() );
143 : 0 : QStringList missingOriginalFields;
144 : 0 : QStringList missingRevisedFields;
145 : 0 : for ( const QString &field : mFieldsToCompare )
146 : : {
147 : 0 : const int originalIndex = mOriginal->fields().lookupField( field );
148 : 0 : mOriginalFieldsToCompareIndices.append( originalIndex );
149 : 0 : if ( originalIndex < 0 )
150 : 0 : missingOriginalFields << field;
151 : :
152 : 0 : const int revisedIndex = mRevised->fields().lookupField( field );
153 : 0 : if ( revisedIndex < 0 )
154 : 0 : missingRevisedFields << field;
155 : 0 : mRevisedFieldsToCompareIndices.append( revisedIndex );
156 : : }
157 : :
158 : 0 : if ( !missingOriginalFields.empty() )
159 : 0 : throw QgsProcessingException( QObject::tr( "Original layer missing selected comparison attributes: %1" ).arg( missingOriginalFields.join( ',' ) ) );
160 : 0 : if ( !missingRevisedFields.empty() )
161 : 0 : throw QgsProcessingException( QObject::tr( "Revised layer missing selected comparison attributes: %1" ).arg( missingRevisedFields.join( ',' ) ) );
162 : :
163 : : return true;
164 : 0 : }
165 : :
166 : 0 : QVariantMap QgsDetectVectorChangesAlgorithm::processAlgorithm( const QVariantMap ¶meters, QgsProcessingContext &context, QgsProcessingFeedback *feedback )
167 : : {
168 : 0 : QString unchangedDestId;
169 : 0 : std::unique_ptr< QgsFeatureSink > unchangedSink( parameterAsSink( parameters, QStringLiteral( "UNCHANGED" ), context, unchangedDestId, mOriginal->fields(),
170 : 0 : mOriginal->wkbType(), mOriginal->sourceCrs() ) );
171 : 0 : if ( !unchangedSink && parameters.value( QStringLiteral( "UNCHANGED" ) ).isValid() )
172 : 0 : throw QgsProcessingException( invalidSinkError( parameters, QStringLiteral( "UNCHANGED" ) ) );
173 : :
174 : 0 : QString addedDestId;
175 : 0 : std::unique_ptr< QgsFeatureSink > addedSink( parameterAsSink( parameters, QStringLiteral( "ADDED" ), context, addedDestId, mRevised->fields(),
176 : 0 : mRevised->wkbType(), mRevised->sourceCrs() ) );
177 : 0 : if ( !addedSink && parameters.value( QStringLiteral( "ADDED" ) ).isValid() )
178 : 0 : throw QgsProcessingException( invalidSinkError( parameters, QStringLiteral( "ADDED" ) ) );
179 : :
180 : 0 : QString deletedDestId;
181 : 0 : std::unique_ptr< QgsFeatureSink > deletedSink( parameterAsSink( parameters, QStringLiteral( "DELETED" ), context, deletedDestId, mOriginal->fields(),
182 : 0 : mOriginal->wkbType(), mOriginal->sourceCrs() ) );
183 : 0 : if ( !deletedSink && parameters.value( QStringLiteral( "DELETED" ) ).isValid() )
184 : 0 : throw QgsProcessingException( invalidSinkError( parameters, QStringLiteral( "DELETED" ) ) );
185 : :
186 : : // first iteration: we loop through the entire original layer, building up a spatial index of ALL original geometries
187 : : // and collecting the original geometries themselves along with the attributes to compare
188 : 0 : QgsFeatureRequest request;
189 : 0 : request.setSubsetOfAttributes( mOriginalFieldsToCompareIndices );
190 : :
191 : 0 : QgsFeatureIterator it = mOriginal->getFeatures( request );
192 : :
193 : 0 : double step = mOriginal->featureCount() > 0 ? 100.0 / mOriginal->featureCount() : 0;
194 : 0 : QHash< QgsFeatureId, QgsGeometry > originalGeometries;
195 : 0 : QHash< QgsFeatureId, QgsAttributes > originalAttributes;
196 : 0 : QHash< QgsAttributes, QgsFeatureId > originalNullGeometryAttributes;
197 : 0 : long current = 0;
198 : :
199 : 0 : QgsAttributes attrs;
200 : 0 : attrs.resize( mFieldsToCompare.size() );
201 : :
202 : 0 : QgsSpatialIndex index( it, [&]( const QgsFeature & f )->bool
203 : : {
204 : 0 : if ( feedback->isCanceled() )
205 : 0 : return false;
206 : :
207 : 0 : if ( f.hasGeometry() )
208 : : {
209 : 0 : originalGeometries.insert( f.id(), f.geometry() );
210 : 0 : }
211 : :
212 : 0 : if ( !mFieldsToCompare.empty() )
213 : : {
214 : 0 : int idx = 0;
215 : 0 : for ( int field : mOriginalFieldsToCompareIndices )
216 : : {
217 : 0 : attrs[idx++] = f.attributes().at( field );
218 : : }
219 : 0 : originalAttributes.insert( f.id(), attrs );
220 : 0 : }
221 : :
222 : 0 : if ( !f.hasGeometry() )
223 : : {
224 : 0 : if ( originalNullGeometryAttributes.contains( attrs ) )
225 : : {
226 : 0 : feedback->reportError( QObject::tr( "A non-unique set of comparison attributes was found for "
227 : 0 : "one or more features without geometries - results may be misleading (features %1 and %2)" ).arg( f.id() ).arg( originalNullGeometryAttributes.value( attrs ) ) );
228 : 0 : }
229 : : else
230 : : {
231 : 0 : originalNullGeometryAttributes.insert( attrs, f.id() );
232 : : }
233 : 0 : }
234 : :
235 : : // overall this loop takes about 10% of time
236 : 0 : current++;
237 : 0 : feedback->setProgress( 0.10 * current * step );
238 : 0 : return true;
239 : 0 : } );
240 : :
241 : 0 : QSet<QgsFeatureId> unchangedOriginalIds;
242 : 0 : QSet<QgsFeatureId> addedRevisedIds;
243 : 0 : current = 0;
244 : :
245 : : // second iteration: we loop through ALL revised features, checking whether each is a match for a geometry from the
246 : : // original set. If so, check if the feature is unchanged. If there's no match with the original features, we mark it as an "added" feature
247 : 0 : step = mRevised->featureCount() > 0 ? 100.0 / mRevised->featureCount() : 0;
248 : 0 : QgsFeatureRequest revisedRequest = QgsFeatureRequest().setDestinationCrs( mOriginal->sourceCrs(), context.transformContext() );
249 : 0 : revisedRequest.setSubsetOfAttributes( mRevisedFieldsToCompareIndices );
250 : 0 : it = mRevised->getFeatures( revisedRequest );
251 : 0 : QgsFeature revisedFeature;
252 : 0 : while ( it.nextFeature( revisedFeature ) )
253 : : {
254 : 0 : if ( feedback->isCanceled() )
255 : 0 : break;
256 : :
257 : 0 : int idx = 0;
258 : 0 : for ( int field : mRevisedFieldsToCompareIndices )
259 : : {
260 : 0 : attrs[idx++] = revisedFeature.attributes().at( field );
261 : : }
262 : :
263 : 0 : bool matched = false;
264 : :
265 : 0 : if ( !revisedFeature.hasGeometry() )
266 : : {
267 : 0 : if ( originalNullGeometryAttributes.contains( attrs ) )
268 : : {
269 : : // found a match for feature
270 : 0 : unchangedOriginalIds.insert( originalNullGeometryAttributes.value( attrs ) );
271 : 0 : matched = true;
272 : 0 : }
273 : 0 : }
274 : : else
275 : : {
276 : : // can we match this feature?
277 : 0 : const QList<QgsFeatureId> candidates = index.intersects( revisedFeature.geometry().boundingBox() );
278 : :
279 : : // lazy evaluate -- there may be NO candidates!
280 : 0 : QgsGeometry revised;
281 : :
282 : 0 : for ( const QgsFeatureId candidateId : candidates )
283 : : {
284 : 0 : if ( unchangedOriginalIds.contains( candidateId ) )
285 : : {
286 : : // already matched this original feature
287 : 0 : continue;
288 : : }
289 : :
290 : : // attribute comparison is faster to do first, if desired
291 : 0 : if ( !mFieldsToCompare.empty() )
292 : : {
293 : 0 : if ( attrs != originalAttributes[ candidateId ] )
294 : : {
295 : : // attributes don't match, so candidates is not a match
296 : 0 : continue;
297 : : }
298 : 0 : }
299 : :
300 : 0 : QgsGeometry original = originalGeometries.value( candidateId );
301 : : // lazy evaluation
302 : 0 : if ( revised.isNull() )
303 : : {
304 : 0 : revised = revisedFeature.geometry();
305 : : // drop z/m if not wanted for match
306 : 0 : switch ( mMatchType )
307 : : {
308 : : case Topological:
309 : : {
310 : 0 : revised.get()->dropMValue();
311 : 0 : revised.get()->dropZValue();
312 : 0 : original.get()->dropMValue();
313 : 0 : original.get()->dropZValue();
314 : 0 : break;
315 : : }
316 : :
317 : : case Exact:
318 : 0 : break;
319 : : }
320 : 0 : }
321 : :
322 : 0 : bool geometryMatch = false;
323 : 0 : switch ( mMatchType )
324 : : {
325 : : case Topological:
326 : : {
327 : 0 : geometryMatch = revised.isGeosEqual( original );
328 : 0 : break;
329 : : }
330 : :
331 : : case Exact:
332 : 0 : geometryMatch = revised.equals( original );
333 : 0 : break;
334 : : }
335 : :
336 : 0 : if ( geometryMatch )
337 : : {
338 : : // candidate is a match for feature
339 : 0 : unchangedOriginalIds.insert( candidateId );
340 : 0 : matched = true;
341 : 0 : break;
342 : : }
343 : 0 : }
344 : 0 : }
345 : :
346 : 0 : if ( !matched )
347 : : {
348 : : // new feature
349 : 0 : addedRevisedIds.insert( revisedFeature.id() );
350 : 0 : }
351 : :
352 : 0 : current++;
353 : 0 : feedback->setProgress( 0.70 * current * step + 10 ); // takes about 70% of time
354 : : }
355 : :
356 : : // third iteration: iterate back over the original features, and direct them to the appropriate sink.
357 : : // If they were marked as unchanged during the second iteration, we put them in the unchanged sink. Otherwise
358 : : // they are placed into the deleted sink.
359 : 0 : step = mOriginal->featureCount() > 0 ? 100.0 / mOriginal->featureCount() : 0;
360 : :
361 : 0 : request = QgsFeatureRequest().setFlags( QgsFeatureRequest::NoGeometry );
362 : 0 : it = mOriginal->getFeatures( request );
363 : 0 : current = 0;
364 : 0 : long deleted = 0;
365 : 0 : QgsFeature f;
366 : 0 : while ( it.nextFeature( f ) )
367 : : {
368 : 0 : if ( feedback->isCanceled() )
369 : 0 : break;
370 : :
371 : : // use already fetched geometry
372 : 0 : f.setGeometry( originalGeometries.value( f.id(), QgsGeometry() ) );
373 : :
374 : 0 : if ( unchangedOriginalIds.contains( f.id() ) )
375 : : {
376 : : // unchanged
377 : 0 : if ( unchangedSink )
378 : 0 : unchangedSink->addFeature( f, QgsFeatureSink::FastInsert );
379 : 0 : }
380 : : else
381 : : {
382 : : // deleted feature
383 : 0 : if ( deletedSink )
384 : 0 : deletedSink->addFeature( f, QgsFeatureSink::FastInsert );
385 : 0 : deleted++;
386 : : }
387 : :
388 : 0 : current++;
389 : 0 : feedback->setProgress( 0.10 * current * step + 80 ); // takes about 10% of time
390 : : }
391 : :
392 : : // forth iteration: collect all added features and add them to the added sink
393 : : // NOTE: while we could potentially do this as part of the second iteration and save some time, we instead
394 : : // do this here using a brand new request because the second iteration
395 : : // is fetching reprojected features and we ideally want geometries from the revised layer's actual CRS only here!
396 : : // also, the second iteration is only fetching the actual attributes used in the comparison, whereas we want
397 : : // to include all attributes in the "added" output
398 : 0 : if ( addedSink )
399 : : {
400 : 0 : step = addedRevisedIds.size() > 0 ? 100.0 / addedRevisedIds.size() : 0;
401 : 0 : it = mRevised->getFeatures( QgsFeatureRequest().setFilterFids( addedRevisedIds ) );
402 : 0 : current = 0;
403 : 0 : while ( it.nextFeature( f ) )
404 : : {
405 : 0 : if ( feedback->isCanceled() )
406 : 0 : break;
407 : :
408 : : // added feature
409 : 0 : addedSink->addFeature( f, QgsFeatureSink::FastInsert );
410 : :
411 : 0 : current++;
412 : 0 : feedback->setProgress( 0.10 * current * step + 90 ); // takes about 10% of time
413 : : }
414 : 0 : }
415 : 0 : feedback->setProgress( 100 );
416 : :
417 : 0 : feedback->pushInfo( QObject::tr( "%1 features unchanged" ).arg( unchangedOriginalIds.size() ) );
418 : 0 : feedback->pushInfo( QObject::tr( "%1 features added" ).arg( addedRevisedIds.size() ) );
419 : 0 : feedback->pushInfo( QObject::tr( "%1 features deleted" ).arg( deleted ) );
420 : :
421 : 0 : QVariantMap outputs;
422 : 0 : outputs.insert( QStringLiteral( "UNCHANGED" ), unchangedDestId );
423 : 0 : outputs.insert( QStringLiteral( "ADDED" ), addedDestId );
424 : 0 : outputs.insert( QStringLiteral( "DELETED" ), deletedDestId );
425 : 0 : outputs.insert( QStringLiteral( "UNCHANGED_COUNT" ), static_cast< long long >( unchangedOriginalIds.size() ) );
426 : 0 : outputs.insert( QStringLiteral( "ADDED_COUNT" ), static_cast< long long >( addedRevisedIds.size() ) );
427 : 0 : outputs.insert( QStringLiteral( "DELETED_COUNT" ), static_cast< long long >( deleted ) );
428 : :
429 : 0 : return outputs;
430 : 0 : }
431 : :
432 : : ///@endcond
|
