(A draft PR hoping to get early feedback.)
- build a DOM tree for: Node that are marked as salient, and GUI Controls. In this way, the screen reader can read the page content.
- make the DOM elements interactive for actionable nodes. So the user can use "tab" key to focus on interactive nodes or controls, and "enter" key to trigger the interactive nodes.
- when the scene content changes (e.g. show/hide new salient mesh), the DOM tree updates too.
@RaananW @Exolun

Testing:
Testing code 1:
Given a scene that has objects that cover conditions of:
- some are salient, some are not not-salient
- some objects (spheres) are interactive, and could be left and right clicked
- objects have parent-children hierarchy
let createScene = function () {
let scene = new BABYLON.Scene(engine);
let camera = new BABYLON.ArcRotateCamera("Camera", Math.PI/2, Math.PI/4, 10, new BABYLON.Vector3(0, 0, 0), scene);
camera.setTarget(BABYLON.Vector3.Zero());
camera.attachControl(canvas, true);
let light = new BABYLON.HemisphericLight("light", new BABYLON.Vector3(0, 1, 0), scene);
light.intensity = 0.7;
// add some objects
// not salient objects
const parent = new BABYLON.TransformNode('parent');
parent.accessibilityTag = {
isSalient: true,
description: "A parent of all, of the root",
}
const boxDecs = new BABYLON.TransformNode('boxDecs');
boxDecs.parent = parent;
boxDecs.accessibilityTag = {
isSalient: true,
description: "A parent without salient children",
}
let boxDec1 = BABYLON.MeshBuilder.CreateBox("boxDec1", {size: 0.3}, scene);
boxDec1.parent = boxDecs;
boxDec1.position.x = -3;
boxDec1.position.z = -4;
let boxDec2 = BABYLON.MeshBuilder.CreateBox("boxDec2", {size: 0.3}, scene);
boxDec2.parent = boxDecs;
boxDec2.position.x = 3;
boxDec2.position.z = -4;
// salient objects, static
let boxes = new BABYLON.TransformNode("boxes");
boxes.parent = parent;
boxes.accessibilityTag = {
isSalient: true,
description: "A group of boxes",
}
let box0 = BABYLON.MeshBuilder.CreateBox("box3", {size: 0.5}, scene);
box0.parent = boxes;
box0.position.z = -1;
box0.position.y = 0.6;
box0.accessibilityTag = {
isSalient: true,
description: "A small box in the middle of the scene",
}
let box1 = BABYLON.MeshBuilder.CreateBox("box1", {}, scene);
box1.parent = boxes;
box1.position.x = 1;
box1.accessibilityTag = {
isSalient: true,
description: "A big box on the left of the small box",
}
let box2 = BABYLON.MeshBuilder.CreateBox("box2", {}, scene);
box2.parent = boxes;
box2.position.x = -1;
box2.accessibilityTag = {
isSalient: true,
description: "A big box on the right of the small box",
}
// salient objects, interactive
let sphereWrapper = new BABYLON.TransformNode('sphereWrapper');
sphereWrapper.accessibilityTag = {
isSalient: true,
description: 'A group of spheres',
};
// sphereWrapper.parent = parent;
let mat = new BABYLON.StandardMaterial("Gray", scene);
mat.diffuseColor = BABYLON.Color3.Gray();
let spheresCount = 6;
let alpha = 0;
for (let index = 0; index < spheresCount; index++) {
const sphere = BABYLON.Mesh.CreateSphere("Sphere " + index, 32, 1, scene);
sphere.parent = sphereWrapper;
sphere.position.x = 3 * Math.cos(alpha);
sphere.position.z = 3 * Math.sin(alpha);
sphere.material = mat;
const sphereOnClicked = () => {
alert(`You just clicked ${sphere.name}!`)
}
const sphereOnLeftClicked2 = () => {
alert(`You just LEFT clicked ${sphere.name}!`)
}
const sphereOnRightClicked = () => {
alert(`You just RIGHT clicked ${sphere.name}!`)
}
sphere.accessibilityTag = {
isSalient: true,
description: sphere.name,
};
sphere.actionManager = new BABYLON.ActionManager(scene);
sphere.actionManager.registerAction(new BABYLON.ExecuteCodeAction(BABYLON.ActionManager.OnPickTrigger, sphereOnClicked));
sphere.actionManager.registerAction(new BABYLON.ExecuteCodeAction(BABYLON.ActionManager.OnLeftPickTrigger, sphereOnLeftClicked2));
sphere.actionManager.registerAction(new BABYLON.ExecuteCodeAction(BABYLON.ActionManager.OnRightPickTrigger, sphereOnRightClicked));
alpha += (2 * Math.PI) / spheresCount;
}
console.log('Start the show!');
AccessibilityRenderer.renderAccessibilityTree(scene);
return scene;
};
Testing code 2:
/**
* A simple GUI scene. Shows a card GUI for a easter event, with buttons to 'Join' event, and 'close' card.
*/
export let createScene = function ()
{
let scene = new BABYLON.Scene(engine);
let camera = new BABYLON.ArcRotateCamera('Camera', -Math.PI/2, Math.PI/2, 5, new BABYLON.Vector3(0, 0, 0), scene);
camera.setTarget(BABYLON.Vector3.Zero());
camera.attachControl(canvas, true);
let light = new BABYLON.HemisphericLight('light', new BABYLON.Vector3(0, 1, 0), scene);
light.intensity = 0.7;
let wrapper = new BABYLON.TransformNode("Wrapper");
wrapper.setEnabled(true);
let egg = BABYLON.MeshBuilder.CreateSphere("Egg", {diameterX: 0.62, diameterY: 0.8, diameterZ: 0.6}, scene);
egg.parent = wrapper;
egg.accessibilityTag = {
isSalient: true,
description: "An easter egg"
}
egg.actionManager = new BABYLON.ActionManager(scene);
egg.actionManager.registerAction(new BABYLON.ExecuteCodeAction(
BABYLON.ActionManager.OnPickTrigger,
() => {
wrapper.setEnabled(false);
card.setEnabled(true);
})
);
let box1 = BABYLON.MeshBuilder.CreateBox("box1", {size: 0.3}, scene);
box1.parent = wrapper;
box1.position.x = 1;
box1.position.z = 0.2;
box1.accessibilityTag = {
isSalient: true,
description: "A small box on the left of the egg",
isWholeObject: true,
}
let box2 = BABYLON.MeshBuilder.CreateBox("box2", {size: 0.3}, scene);
box2.parent = wrapper;
box2.position.x = -1;
box2.position.z = 0.2;
box2.accessibilityTag = {
isSalient: true,
description: "A small box on the right of the egg",
isWholeObject: true,
}
let box = BABYLON.MeshBuilder.CreateBox("box", {size: 0.5}, scene);
box.position.y = -0.65;
box.parent = egg;
// GUI
let card = BABYLON.MeshBuilder.CreatePlane('card', {size: 3});
card.setEnabled(false);
card.accessibilityTag = {
isSalient: true,
description: "Easter Event Card"
}
card.position.z = 0.5;
let adt = GUI.AdvancedDynamicTexture.CreateForMesh(card);
let wrapFront = new GUI.Rectangle('TeamsCardUI_wrapFront');
wrapFront.width = '80%';
wrapFront.background = 'white';
adt.addControl(wrapFront);
let thumbnailBg = new GUI.Rectangle('TeamsCardUI_ThumbnailBg');
thumbnailBg.width = '100%';
thumbnailBg.height = '40%';
thumbnailBg.background = 'gray';
thumbnailBg.verticalAlignment = GUI.Control.VERTICAL_ALIGNMENT_TOP;
thumbnailBg.top = 0;
wrapFront.addControl(thumbnailBg);
let url = 'https://raw.githubusercontent.com/TNyawara/EasterBunnyGroupProject/master/Assets/Easter_UI/Backgrounds/background_10.png';
let thumbnailCustomized = new GUI.Image('TeamsCardUI_ThumbnailImage', url);
thumbnailCustomized.alt = 'Background image';
thumbnailCustomized.width = '100%';
thumbnailCustomized.height = '100%';
thumbnailBg.addControl(thumbnailCustomized);
const titleFront = new GUI.TextBlock(
'TeamsCardUIText_Title',
'Event: Happy Hoppy'
);
titleFront.fontSize = 60;
titleFront.verticalAlignment = GUI.Control.VERTICAL_ALIGNMENT_TOP;
titleFront.textHorizontalAlignment = GUI.Control.HORIZONTAL_ALIGNMENT_LEFT;
titleFront.paddingLeft = '7.5%';
titleFront.top = '40%';
titleFront.height = '10%';
wrapFront.addControl(titleFront);
let dateFront = new GUI.TextBlock('TeamsCardUIText_Date', 'Every day');
wrapFront.addControl(dateFront);
dateFront.fontSize = 40;
dateFront.verticalAlignment = GUI.Control.VERTICAL_ALIGNMENT_TOP;
dateFront.textHorizontalAlignment = GUI.Control.HORIZONTAL_ALIGNMENT_LEFT;
dateFront.paddingLeft = '7.5%';
dateFront.top = '50%';
dateFront.height = '5%';
dateFront.isEnabled = false;
const timeFront = new GUI.TextBlock('TeamsCardUIText_Time', '00:00 - 23:59');
wrapFront.addControl(timeFront);
timeFront.fontSize = 40;
timeFront.verticalAlignment = GUI.Control.VERTICAL_ALIGNMENT_TOP;
timeFront.textHorizontalAlignment = GUI.Control.HORIZONTAL_ALIGNMENT_LEFT;
timeFront.paddingLeft = '35%';
timeFront.top = '50%';
timeFront.height = '5%';
const meetingDetail = new GUI.TextBlock(
'TeamsCardUIText_GroupName',
"Help the little bunny rabbits get ready for Easter! Look at all the different colors to decorate Easter eggs with and pick out the shapes you'd like to wear in the parade. "
);
wrapFront.addControl(meetingDetail);
meetingDetail.fontSize = 40;
meetingDetail.verticalAlignment = GUI.Control.VERTICAL_ALIGNMENT_TOP;
meetingDetail.textHorizontalAlignment = GUI.Control.HORIZONTAL_ALIGNMENT_LEFT;
meetingDetail.paddingLeft = '7.5%';
meetingDetail.top = '55%';
meetingDetail.height = '30%';
meetingDetail.width = '100%';
meetingDetail.textWrapping = GUI.TextWrapping.WordWrapEllipsis;
let joinButton = GUI.Button.CreateSimpleButton('TeamsCardUIButton_Join', 'Join');
joinButton.background = 'black';
joinButton.color = 'white';
joinButton.fontSize = 40;
joinButton.verticalAlignment = GUI.Control.VERTICAL_ALIGNMENT_TOP;
joinButton.textBlock.textHorizontalAlignment = GUI.Control.VERTICAL_ALIGNMENT_CENTER;
joinButton.top = '85%';
joinButton.height = '10%';
joinButton.width = '40%';
wrapFront.addControl(joinButton);
joinButton.onPointerClickObservable.add(() => {
alert('💐🌼Happy Easter! 🐰🥚');
});
let closeButton = GUI.Button.CreateSimpleButton('TeamsCardUIButton_Close', 'X');
closeButton.background = 'black';
closeButton.color = 'white';
closeButton.fontSize = 40;
closeButton.verticalAlignment = GUI.Control.VERTICAL_ALIGNMENT_TOP;
closeButton.horizontalAlignment = GUI.Control.HORIZONTAL_ALIGNMENT_RIGHT;
closeButton.textBlock.textHorizontalAlignment = GUI.Control.VERTICAL_ALIGNMENT_CENTER;
closeButton.top = '0';
closeButton.height = '12%';
closeButton.width = '15%';
wrapFront.addControl(closeButton);
closeButton.onPointerClickObservable.add(() => {
card.setEnabled(false);
wrapper.setEnabled(true);
});
console.log('Start the show!');
AccessibilityRenderer.RenderAccessibilityTree(scene);
return scene;
}