Skip to content

Commit d430bff

Browse files
weizijunweizijun
authored
Downsampling: remove null value setting (elastic#90025)
Do not copy the total fields limit setting into the downsampled index if the value is null in the source index.
1 parent 369dca3 commit d430bff

File tree

2 files changed

+66
-62
lines changed

2 files changed

+66
-62
lines changed

‎x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportRollupAction.java

+23-22
Original file line numberDiff line numberDiff line change
@@ -594,32 +594,33 @@ private void createRollupIndex(
594594
DownsampleAction.Request request,
595595
ActionListener<AcknowledgedResponse> listener
596596
) {
597+
/*
598+
* When creating the rollup index, we copy the index.number_of_shards from source index,
599+
* and we set the index.number_of_replicas to 0, to avoid replicating the index being built.
600+
* Also, we set the index.refresh_interval to -1.
601+
* We will set the correct number of replicas and refresh the index later.
602+
*
603+
* We should note that there is a risk of losing a node during the rollup process. In this
604+
* case rollup will fail.
605+
*/
606+
Settings.Builder builder = Settings.builder()
607+
.put(IndexMetadata.SETTING_INDEX_HIDDEN, true)
608+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, sourceIndexMetadata.getNumberOfShards())
609+
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
610+
.put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "-1")
611+
.put(IndexMetadata.INDEX_DOWNSAMPLE_STATUS.getKey(), IndexMetadata.DownsampleTaskStatus.STARTED);
612+
if (sourceIndexMetadata.getSettings().hasValue(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey())) {
613+
builder.put(
614+
MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(),
615+
sourceIndexMetadata.getSettings().get(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey())
616+
);
617+
}
618+
597619
CreateIndexClusterStateUpdateRequest createIndexClusterStateUpdateRequest = new CreateIndexClusterStateUpdateRequest(
598620
"rollup",
599621
rollupIndexName,
600622
rollupIndexName
601-
).settings(
602-
/*
603-
* When creating the rollup index, we copy the index.number_of_shards from source index,
604-
* and we set the index.number_of_replicas to 0, to avoid replicating the index being built.
605-
* Also, we set the index.refresh_interval to -1.
606-
* We will set the correct number of replicas and refresh the index later.
607-
*
608-
* We should note that there is a risk of losing a node during the rollup process. In this
609-
* case rollup will fail.
610-
*/
611-
Settings.builder()
612-
.put(IndexMetadata.SETTING_INDEX_HIDDEN, true)
613-
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, sourceIndexMetadata.getNumberOfShards())
614-
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
615-
.put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "-1")
616-
.put(IndexMetadata.INDEX_DOWNSAMPLE_STATUS.getKey(), IndexMetadata.DownsampleTaskStatus.STARTED)
617-
.put(
618-
MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(),
619-
sourceIndexMetadata.getSettings().get(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey())
620-
)
621-
.build()
622-
).mappings(mapping);
623+
).settings(builder.build()).mappings(mapping);
623624
clusterService.submitStateUpdateTask("create-rollup-index [" + rollupIndexName + "]", new RollupClusterStateUpdateTask(listener) {
624625
@Override
625626
public ClusterState execute(ClusterState currentState) throws Exception {

‎x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java

+43-40
Original file line numberDiff line numberDiff line change
@@ -835,70 +835,73 @@ private void assertFieldMappings(
835835
}
836836

837837
private void assertRollupIndexSettings(String sourceIndex, String rollupIndex, GetIndexResponse indexSettingsResp) {
838+
Settings sourceSettings = indexSettingsResp.settings().get(sourceIndex);
839+
Settings rollupSettings = indexSettingsResp.settings().get(rollupIndex);
840+
838841
// Assert rollup metadata are set in index settings
839-
assertEquals("success", indexSettingsResp.getSetting(rollupIndex, IndexMetadata.INDEX_DOWNSAMPLE_STATUS_KEY));
842+
assertEquals("success", rollupSettings.get(IndexMetadata.INDEX_DOWNSAMPLE_STATUS_KEY));
840843

841-
assertNotNull(indexSettingsResp.getSetting(sourceIndex, IndexMetadata.SETTING_INDEX_UUID));
842-
assertNotNull(indexSettingsResp.getSetting(rollupIndex, IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_UUID_KEY));
844+
assertNotNull(sourceSettings.get(IndexMetadata.SETTING_INDEX_UUID));
845+
assertNotNull(rollupSettings.get(IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_UUID_KEY));
843846
assertEquals(
844-
indexSettingsResp.getSetting(sourceIndex, IndexMetadata.SETTING_INDEX_UUID),
845-
indexSettingsResp.getSetting(rollupIndex, IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_UUID_KEY)
847+
sourceSettings.get(IndexMetadata.SETTING_INDEX_UUID),
848+
rollupSettings.get(IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_UUID_KEY)
846849
);
847850

848-
assertEquals(sourceIndex, indexSettingsResp.getSetting(rollupIndex, IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_NAME_KEY));
849-
assertEquals(
850-
indexSettingsResp.getSetting(sourceIndex, IndexSettings.MODE.getKey()),
851-
indexSettingsResp.getSetting(rollupIndex, IndexSettings.MODE.getKey())
852-
);
851+
assertEquals(sourceIndex, rollupSettings.get(IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_NAME_KEY));
852+
assertEquals(sourceSettings.get(IndexSettings.MODE.getKey()), rollupSettings.get(IndexSettings.MODE.getKey()));
853853

854-
assertNotNull(indexSettingsResp.getSetting(sourceIndex, IndexSettings.TIME_SERIES_START_TIME.getKey()));
855-
assertNotNull(indexSettingsResp.getSetting(rollupIndex, IndexSettings.TIME_SERIES_START_TIME.getKey()));
854+
assertNotNull(sourceSettings.get(IndexSettings.TIME_SERIES_START_TIME.getKey()));
855+
assertNotNull(rollupSettings.get(IndexSettings.TIME_SERIES_START_TIME.getKey()));
856856
assertEquals(
857-
indexSettingsResp.getSetting(sourceIndex, IndexSettings.TIME_SERIES_START_TIME.getKey()),
858-
indexSettingsResp.getSetting(rollupIndex, IndexSettings.TIME_SERIES_START_TIME.getKey())
857+
sourceSettings.get(IndexSettings.TIME_SERIES_START_TIME.getKey()),
858+
rollupSettings.get(IndexSettings.TIME_SERIES_START_TIME.getKey())
859859
);
860860

861-
assertNotNull(indexSettingsResp.getSetting(sourceIndex, IndexSettings.TIME_SERIES_END_TIME.getKey()));
862-
assertNotNull(indexSettingsResp.getSetting(rollupIndex, IndexSettings.TIME_SERIES_END_TIME.getKey()));
861+
assertNotNull(sourceSettings.get(IndexSettings.TIME_SERIES_END_TIME.getKey()));
862+
assertNotNull(rollupSettings.get(IndexSettings.TIME_SERIES_END_TIME.getKey()));
863863
assertEquals(
864-
indexSettingsResp.getSetting(sourceIndex, IndexSettings.TIME_SERIES_END_TIME.getKey()),
865-
indexSettingsResp.getSetting(rollupIndex, IndexSettings.TIME_SERIES_END_TIME.getKey())
864+
sourceSettings.get(IndexSettings.TIME_SERIES_END_TIME.getKey()),
865+
rollupSettings.get(IndexSettings.TIME_SERIES_END_TIME.getKey())
866866
);
867-
assertNotNull(indexSettingsResp.getSetting(sourceIndex, IndexMetadata.INDEX_ROUTING_PATH.getKey()));
868-
assertNotNull(indexSettingsResp.getSetting(rollupIndex, IndexMetadata.INDEX_ROUTING_PATH.getKey()));
867+
assertNotNull(sourceSettings.get(IndexMetadata.INDEX_ROUTING_PATH.getKey()));
868+
assertNotNull(rollupSettings.get(IndexMetadata.INDEX_ROUTING_PATH.getKey()));
869869
assertEquals(
870-
indexSettingsResp.getSetting(sourceIndex, IndexMetadata.INDEX_ROUTING_PATH.getKey()),
871-
indexSettingsResp.getSetting(rollupIndex, IndexMetadata.INDEX_ROUTING_PATH.getKey())
870+
sourceSettings.get(IndexMetadata.INDEX_ROUTING_PATH.getKey()),
871+
rollupSettings.get(IndexMetadata.INDEX_ROUTING_PATH.getKey())
872872
);
873873

874-
assertNotNull(indexSettingsResp.getSetting(sourceIndex, IndexMetadata.SETTING_NUMBER_OF_SHARDS));
875-
assertNotNull(indexSettingsResp.getSetting(rollupIndex, IndexMetadata.SETTING_NUMBER_OF_SHARDS));
874+
assertNotNull(sourceSettings.get(IndexMetadata.SETTING_NUMBER_OF_SHARDS));
875+
assertNotNull(rollupSettings.get(IndexMetadata.SETTING_NUMBER_OF_SHARDS));
876876
assertEquals(
877-
indexSettingsResp.getSetting(sourceIndex, IndexMetadata.SETTING_NUMBER_OF_SHARDS),
878-
indexSettingsResp.getSetting(rollupIndex, IndexMetadata.SETTING_NUMBER_OF_SHARDS)
877+
sourceSettings.get(IndexMetadata.SETTING_NUMBER_OF_SHARDS),
878+
rollupSettings.get(IndexMetadata.SETTING_NUMBER_OF_SHARDS)
879879
);
880880

881-
assertNotNull(indexSettingsResp.getSetting(sourceIndex, IndexMetadata.SETTING_NUMBER_OF_REPLICAS));
882-
assertNotNull(indexSettingsResp.getSetting(rollupIndex, IndexMetadata.SETTING_NUMBER_OF_REPLICAS));
883-
assertEquals(
884-
indexSettingsResp.getSetting(sourceIndex, IndexMetadata.SETTING_NUMBER_OF_REPLICAS),
885-
indexSettingsResp.getSetting(rollupIndex, IndexMetadata.SETTING_NUMBER_OF_REPLICAS)
886-
);
887-
assertEquals("true", indexSettingsResp.getSetting(rollupIndex, IndexMetadata.SETTING_BLOCKS_WRITE));
881+
assertNotNull(sourceSettings.get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS));
882+
assertNotNull(rollupSettings.get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS));
888883
assertEquals(
889-
indexSettingsResp.getSetting(sourceIndex, IndexMetadata.SETTING_INDEX_HIDDEN),
890-
indexSettingsResp.getSetting(rollupIndex, IndexMetadata.SETTING_INDEX_HIDDEN)
884+
sourceSettings.get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS),
885+
rollupSettings.get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS)
891886
);
892887

893-
if (indexSettingsResp.getSetting(sourceIndex, MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey()) != null) {
888+
assertEquals("true", rollupSettings.get(IndexMetadata.SETTING_BLOCKS_WRITE));
889+
assertEquals(sourceSettings.get(IndexMetadata.SETTING_INDEX_HIDDEN), rollupSettings.get(IndexMetadata.SETTING_INDEX_HIDDEN));
890+
891+
if (sourceSettings.keySet().contains(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey())) {
894892
assertNotNull(indexSettingsResp.getSetting(rollupIndex, MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey()));
895893
}
896-
if (indexSettingsResp.getSetting(sourceIndex, MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey()) != null
897-
&& indexSettingsResp.getSetting(rollupIndex, MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey()) != null)
894+
895+
if (sourceSettings.keySet().contains(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey())
896+
&& rollupSettings.keySet().contains(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey())) {
898897
assertEquals(
899-
indexSettingsResp.getSetting(sourceIndex, MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey()),
900-
indexSettingsResp.getSetting(rollupIndex, MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey())
898+
sourceSettings.get(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey()),
899+
rollupSettings.get(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey())
901900
);
901+
} else {
902+
assertFalse(sourceSettings.keySet().contains(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey()));
903+
assertFalse(rollupSettings.keySet().contains(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey()));
904+
}
902905
}
903906

904907
private AggregationBuilder buildAggregations(

0 commit comments

Comments
 (0)