Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Incremental Training Phase 3 Implementation #450

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
+2,996 −2,070
Open

Incremental Training Phase 3 Implementation #450

Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
b97f360
Add incremental training and related unit tests
yunboouyang Oct 24, 2019
d5d9ac9
Add calculation details in unit tests and fix integ test failure
yunboouyang Oct 29, 2019
eecae47
Revise incremental training code to test via GAME training template
yunboouyang Nov 13, 2019
1d00902
Enable Autotune for Incremental Training and Fix Subspace projection …
yunboouyang Jan 10, 2020
d67bb74
Remove L1 regularization
yunboouyang Jan 23, 2020
73b475f
Add full Hessian matrix calculation
yunboouyang Jan 31, 2020
0fc80a4
Fix a RandomEffectCoordinate bug
yunboouyang Feb 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion ...api/src/integTest/scala/com/linkedin/photon/ml/algorithm/CoordinateFactoryIntegTest.scala
View file
Original file line number Diff line number Diff line change
@@ -23,6 +23,7 @@ import com.linkedin.photon.ml.TaskType
import com.linkedin.photon.ml.Types.REId
import com.linkedin.photon.ml.data.{FixedEffectDataset, LocalDataset, RandomEffectDataset}
import com.linkedin.photon.ml.function.{DistributedObjectiveFunction, ObjectiveFunctionHelper, SingleNodeObjectiveFunction}
import com.linkedin.photon.ml.model.{FixedEffectModel, RandomEffectModel}
import com.linkedin.photon.ml.normalization.NormalizationContext
import com.linkedin.photon.ml.optimization.game.{FixedEffectOptimizationConfiguration, RandomEffectOptimizationConfiguration}
import com.linkedin.photon.ml.optimization.{OptimizerConfig, OptimizerType, SingleNodeOptimizationProblem, VarianceComputationType}
@@ -46,6 +47,7 @@ class CoordinateFactoryIntegTest extends SparkTestUtils {

val mockDataset = mock(classOf[FixedEffectDataset])
val optimizationConfiguration = FixedEffectOptimizationConfiguration(OPTIMIZER_CONFIG)
val priorModelOpt: Option[FixedEffectModel] = None

doReturn(sc).when(mockDataset).sparkContext

@@ -57,6 +59,7 @@ class CoordinateFactoryIntegTest extends SparkTestUtils {
DOWN_SAMPLER_FACTORY,
MOCK_NORMALIZATION,
VARIANCE_COMPUTATION_TYPE,
priorModelOpt,
INTERCEPT_INDEX)

coordinate match {
@@ -78,8 +81,10 @@ class CoordinateFactoryIntegTest extends SparkTestUtils {
val mockProjectorsRDD = mock(classOf[RDD[(REId, LinearSubspaceProjector)]])
val mockProblemsRDD = mock(classOf[RDD[(REId, SingleNodeOptimizationProblem[SingleNodeObjectiveFunction])]])
val optimizationConfiguration = RandomEffectOptimizationConfiguration(OPTIMIZER_CONFIG)
val priorModelOpt: Option[RandomEffectModel] = None

doReturn(sc).when(mockDataset).sparkContext
doReturn(sc).when(mockProjectorsRDD).sparkContext
doReturn(mockDataRDD).when(mockDataset).activeData
doReturn(mockDataRDD)
.when(mockDataRDD)
@@ -97,6 +102,7 @@ class CoordinateFactoryIntegTest extends SparkTestUtils {
DOWN_SAMPLER_FACTORY,
MOCK_NORMALIZATION,
VARIANCE_COMPUTATION_TYPE,
priorModelOpt,
INTERCEPT_INDEX)

coordinate match {
@@ -124,6 +130,7 @@ class CoordinateFactoryIntegTest extends SparkTestUtils {
DOWN_SAMPLER_FACTORY,
MOCK_NORMALIZATION,
VARIANCE_COMPUTATION_TYPE,
None,
INTERCEPT_INDEX)
}
}
@@ -139,7 +146,7 @@ object CoordinateFactoryIntegTest {
private val INTERCEPT_INDEX = None

private val OPTIMIZER_CONFIG = OptimizerConfig(OPTIMIZER_TYPE, MAX_ITER, TOLERANCE)
private val MOCK_NORMALIZATION = mock(classOf[NormalizationContext])
private val MOCK_NORMALIZATION = mock(classOf[NormalizationContext], withSettings().serializable())
private val GLM_CONSTRUCTOR = LogisticRegressionModel.apply _
private val LOSS_FUNCTION_FACTORY = ObjectiveFunctionHelper.buildFactory(TRAINING_TASK, TREE_AGGREGATE_DEPTH)
private val DOWN_SAMPLER_FACTORY = DownSamplerHelper.buildFactory(TRAINING_TASK)
4 changes: 3 additions & 1 deletion ...on-api/src/integTest/scala/com/linkedin/photon/ml/data/RandomEffectDatasetIntegTest.scala
View file
Original file line number Diff line number Diff line change
@@ -184,7 +184,7 @@ class RandomEffectDatasetIntegTest extends SparkTestUtils {
val partitioner = new RandomEffectDatasetPartitioner(NUM_PARTITIONS, sc.broadcast(partitionMap))

val projectorsMap = RandomEffectDataset
.generateLinearSubspaceProjectors(keyedGameDatasetRDD, partitioner)
.generateLinearSubspaceProjectors(keyedGameDatasetRDD, partitioner, None)
.collect
.toMap

@@ -381,6 +381,7 @@ class RandomEffectDatasetIntegTest extends SparkTestUtils {
NUM_PARTITIONS)
val randomEffectDataset = RandomEffectDataset(
dataRDD,
None,
randomEffectDataConfig,
rePartitioner,
None,
@@ -440,6 +441,7 @@ class RandomEffectDatasetIntegTest extends SparkTestUtils {
Some(activeDataLowerBound))
val randomEffectDataset = RandomEffectDataset(
dataRDD,
None,
randomEffectDataConfig,
rePartitioner,
None,
Loading