Dealing with multiple sources

import argparse
import otbtf
import keras
from mymetrics import FScore


class_nb = 6  # number of classes
inp1_key = "input_10m"  # model input for 10m bands
inp2_key = "input_20m"  # model input for 20m bands
tgt_key = "estimated"  # model target


def dataset_preprocessing_fn(sample):
    return {
        inp1_key: sample["img_10m"],
        inp2_key: sample["img_20m"],
        tgt_key: otbtf.ops.one_hot(labels=sample["labels"], nb_classes=class_nb),
    }


def create_dataset(img_10m, img_20m, labels, batch_size=8):
    otbtf_dataset = otbtf.DatasetFromPatchesImages(
        filenames_dict={"img_10m": img_10m, "img_20m": img_20m, "labels": labels}
    )
    return otbtf_dataset.get_tf_dataset(
        batch_size=batch_size,
        preprocessing_fn=dataset_preprocessing_fn,
        targets_keys=[tgt_key],
    )


# Training dataset
ds_train = create_dataset(
    ["/data/a_img_10m_2src.tif"],
    ["/data/a_img_20m_2src.tif"],
    ["/data/a_labels_2src.tif"],
)
ds_train = ds_train.shuffle(buffer_size=100)

# Validation dataset
ds_valid = create_dataset(
    ["/data/b_img_10m_2src.tif"],
    ["/data/b_img_20m_2src.tif"],
    ["/data/b_labels_2src.tif"],
)


def conv(inp, depth, kernel_size, name, activation="relu"):
    conv_op = keras.layers.Conv2D(
        filters=depth,
        kernel_size=kernel_size,
        strides=1,
        activation=activation,
        padding="valid",
        name=name,
    )
    return conv_op(inp)


pool = keras.layers.MaxPool2D(pool_size=(2, 2))


class DualSrcFCNNModel(otbtf.ModelBase):
    """ " This is a subclass of `otbtf.ModelBase` to implement a CNN."""

    def normalize_inputs(self, inputs):
        """This function nomalizes both inputs, scaling values by 1e-4."""
        return {
            key: keras.ops.cast(inputs[key], "float32") * 1e-4
            for key in [inp1_key, inp2_key]
        }

    def get_outputs(self, normalized_inputs):
        """This function implements the model"""

        # The 10m branch
        inp1 = normalized_inputs[inp1_key]  # 16x16x4
        net1 = conv(inp1, 16, 5, "conv11")  # 12x12x16
        net1 = conv(net1, 16, 3, "conv12")  # 10x10x16
        net1 = conv(net1, 16, 3, "conv13")  # 8x8x16
        net1 = conv(net1, 32, 3, "conv14")  # 6x6x32
        net1 = conv(net1, 32, 3, "conv15")  # 4x4x32
        net1 = pool(net1)  # 2x2x32
        net1 = conv(net1, 32, 2, "feats1")  # 1x1x32

        # The 20m branch
        inp2 = normalized_inputs[inp2_key]  # 8x8x6
        net2 = conv(inp2, 32, 3, "conv21")  # 6x6x32
        net2 = conv(net2, 32, 3, "conv22")  # 4x4x32
        net2 = conv(net2, 32, 3, "conv23")  # 2x2x32
        net2 = conv(net2, 32, 2, "feats2")  # 1x1x32

        # Merge features
        net = keras.layers.Concatenate(name="feats", axis=-1)([net1, net2])

        # Classifier
        estim = conv(net, class_nb, 1, "softmax_layer", activation="softmax")
        argmax_op = otbtf.layers.Argmax()

        return {
            tgt_key: estim,
            "estimated_labels": argmax_op(estim),  # additional output: class id
            "features": net,  # additional output: features
        }


parser = argparse.ArgumentParser(description="Train a CNN model")
parser.add_argument("--model", required=True, help="model path (.keras file)")
parser.add_argument("--log_dir", required=True, help="logs directory")
parser.add_argument("--batch_size", type=int, default=4)
parser.add_argument("--learning_rate", type=float, default=0.0002)
parser.add_argument("--epochs", type=int, default=100)
params = parser.parse_args()

model = DualSrcFCNNModel(dataset_element_spec=ds_train.element_spec)

# Precision and recall for each class
metrics = [
    cls(class_id=class_id)
    for class_id in range(class_nb)
    for cls in [keras.metrics.Precision, keras.metrics.Recall]
]

# F1-Score for each class
metrics += [
    FScore(class_id=class_id, name=f"fscore_cls{class_id}")
    for class_id in range(class_nb)
]

model.compile(
    loss={tgt_key: keras.losses.CategoricalCrossentropy()},
    optimizer=keras.optimizers.Adam(params.learning_rate),
    metrics={tgt_key: metrics},  # compute the metrics for `tgt_key`
)
model.summary()

save_callback = keras.callbacks.ModelCheckpoint(
    params.model,  # new directory name
    save_best_only=True,  # save only the best models
    monitor="val_loss",  # metric or loss to monitor
    mode="min",  # when a new min is reached
    verbose=2,  # log something when saving
)
tb_callback = keras.callbacks.TensorBoard(log_dir=params.log_dir)
model.fit(
    ds_train,
    epochs=params.epochs,
    validation_data=ds_valid,
    callbacks=[save_callback, tb_callback],
)

python part_2_train_fcn_2_sources.py \
  --model /data/models/model2_dualsrc.keras \
  --log_dir /data/logs/model2_dualsrc

import pyotb
import argparse

parser = argparse.ArgumentParser(description="Apply the model")
parser.add_argument("--savedmodel", required=True, help="savedmodel directory")
params = parser.parse_args()

# Generate the classification map, patch-based mode, same
# resolution as the 20m spacing image
infer = pyotb.TensorflowModelServe(
    n_sources=2,  # Tells the OTB application to use two sources
    source1_il="/data/s2_tokyo_10m.tif",
    source1_rfieldx=16,
    source1_rfieldy=16,
    source1_placeholder="input_10m",
    source2_il="/data/s2_tokyo_20m.tif",
    source2_rfieldx=8,
    source2_rfieldy=8,
    source2_placeholder="input_20m",
    model_dir=params.savedmodel,
    output_names="estimated_labels",
    output_spcscale=2,
)

infer.write(
    "/data/map_fcn_2_sources_pb_spc2.tif",
    pixel_type="uint8",
    ext_fname="box=2000:2000:500:500",
)

import pyotb
import argparse

parser = argparse.ArgumentParser(description="Apply the model")
parser.add_argument("--savedmodel", required=True, help="savedmodel directory")
params = parser.parse_args()

# Generate the classification map, fully-convolutional mode, same
# resolution as the 20m spacing image
infer = pyotb.TensorflowModelServe(
    n_sources=2,  # Tells the OTB application to use two sources
    source1_il="/data/s2_tokyo_10m.tif",
    source1_rfieldx=16,
    source1_rfieldy=16,
    source1_placeholder="input_10m",
    source2_il="/data/s2_tokyo_20m.tif",
    source2_rfieldx=8,
    source2_rfieldy=8,
    source2_placeholder="input_20m",
    model_dir=params.savedmodel,
    model_fullyconv=True,
    output_names="estimated_labels",
    output_spcscale=2,
)

infer.write(
    "/data/map_fcn_2_sources_fcn_spc2.tif",
    pixel_type="uint8",
    ext_fname="box=2000:2000:500:500",
)