001 /* 002 * Copyright (c) 2007-2014 Concurrent, Inc. All Rights Reserved. 003 * 004 * Project and contact information: http://www.cascading.org/ 005 * 006 * This file is part of the Cascading project. 007 * 008 * Licensed under the Apache License, Version 2.0 (the "License"); 009 * you may not use this file except in compliance with the License. 010 * You may obtain a copy of the License at 011 * 012 * http://www.apache.org/licenses/LICENSE-2.0 013 * 014 * Unless required by applicable law or agreed to in writing, software 015 * distributed under the License is distributed on an "AS IS" BASIS, 016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 017 * See the License for the specific language governing permissions and 018 * limitations under the License. 019 */ 020 021 package cascading.tuple.collect; 022 023 import java.util.ArrayList; 024 import java.util.List; 025 import java.util.Properties; 026 027 import cascading.property.Props; 028 import cascading.util.Util; 029 030 /** 031 * Class SpillableProps is a fluent interface for building properties to be passed to a 032 * {@link cascading.flow.FlowConnector} before creating new {@link cascading.flow.Flow} instances. 033 * 034 * @see SpillableTupleList 035 * @see SpillableTupleMap 036 */ 037 public class SpillableProps extends Props 038 { 039 /** 040 * Whether to enable compress of the spills or not, on by default. 041 * 042 * @see Boolean#parseBoolean(String) 043 */ 044 public static final String SPILL_COMPRESS = "cascading.spill.compress"; 045 046 /** A comma delimited list of possible codecs to try. This is platform dependent. */ 047 public static final String SPILL_CODECS = "cascading.spill.codecs"; 048 049 /** Number of tuples to hold in memory before spilling them to disk. */ 050 public static final String LIST_THRESHOLD = "cascading.spill.list.threshold"; 051 052 /** The total number of tuple values (not keys) to attempt to keep in memory. */ 053 public static final String MAP_THRESHOLD = "cascading.spill.map.threshold"; 054 055 /** 056 * The initial hash map capacity. 057 * 058 * @see java.util.HashMap 059 */ 060 public static final String MAP_CAPACITY = "cascading.spill.map.capacity"; 061 062 /** 063 * The initial hash map load factor. 064 * 065 * @see java.util.HashMap 066 */ 067 public static final String MAP_LOADFACTOR = "cascading.spill.map.loadfactor"; 068 069 public static final int defaultListThreshold = 10 * 1000; 070 071 public static final int defaultMapThreshold = 10 * 1000; 072 public static final int defaultMapInitialCapacity = 100 * 1000; 073 public static final float defaultMapLoadFactor = 0.75f; 074 075 boolean compressSpill = true; 076 List<String> codecs = new ArrayList<String>(); 077 078 int listSpillThreshold = defaultListThreshold; 079 080 int mapSpillThreshold = defaultMapThreshold; 081 int mapInitialCapacity = defaultMapInitialCapacity; 082 float mapLoadFactor = defaultMapLoadFactor; 083 084 public static SpillableProps spillableProps() 085 { 086 return new SpillableProps(); 087 } 088 089 public SpillableProps() 090 { 091 } 092 093 public boolean isCompressSpill() 094 { 095 return compressSpill; 096 } 097 098 public SpillableProps setCompressSpill( boolean compressSpill ) 099 { 100 this.compressSpill = compressSpill; 101 102 return this; 103 } 104 105 public List<String> getCodecs() 106 { 107 return codecs; 108 } 109 110 public SpillableProps setCodecs( List<String> codecs ) 111 { 112 this.codecs = codecs; 113 114 return this; 115 } 116 117 public SpillableProps addCodecs( List<String> codecs ) 118 { 119 this.codecs.addAll( codecs ); 120 121 return this; 122 } 123 124 public SpillableProps addCodec( String codec ) 125 { 126 this.codecs.add( codec ); 127 128 return this; 129 } 130 131 public int getListSpillThreshold() 132 { 133 return listSpillThreshold; 134 } 135 136 public SpillableProps setListSpillThreshold( int listSpillThreshold ) 137 { 138 this.listSpillThreshold = listSpillThreshold; 139 140 return this; 141 } 142 143 public int getMapSpillThreshold() 144 { 145 return mapSpillThreshold; 146 } 147 148 public SpillableProps setMapSpillThreshold( int mapSpillThreshold ) 149 { 150 this.mapSpillThreshold = mapSpillThreshold; 151 152 return this; 153 } 154 155 public int getMapInitialCapacity() 156 { 157 return mapInitialCapacity; 158 } 159 160 public SpillableProps setMapInitialCapacity( int mapInitialCapacity ) 161 { 162 this.mapInitialCapacity = mapInitialCapacity; 163 164 return this; 165 } 166 167 public float getMapLoadFactor() 168 { 169 return mapLoadFactor; 170 } 171 172 public SpillableProps setMapLoadFactor( float mapLoadFactor ) 173 { 174 this.mapLoadFactor = mapLoadFactor; 175 176 return this; 177 } 178 179 @Override 180 protected void addPropertiesTo( Properties properties ) 181 { 182 for( String codec : codecs ) 183 { 184 String codecs = (String) properties.get( SPILL_CODECS ); 185 186 properties.put( SPILL_CODECS, Util.join( ",", Util.removeNulls( codecs, codec ) ) ); 187 } 188 189 properties.setProperty( SPILL_COMPRESS, Boolean.toString( compressSpill ) ); 190 properties.setProperty( LIST_THRESHOLD, Integer.toString( listSpillThreshold ) ); 191 192 properties.setProperty( MAP_THRESHOLD, Integer.toString( mapSpillThreshold ) ); 193 properties.setProperty( MAP_CAPACITY, Integer.toString( mapInitialCapacity ) ); 194 properties.setProperty( MAP_LOADFACTOR, Float.toString( mapLoadFactor ) ); 195 } 196 }