Show:

File: app/components/exp-lookit-images-audio/component.js

  1. import Ember from 'ember';
  2. import layout from './template';
  3. import ExpFrameBaseComponent from '../exp-frame-base/component';
  4. import FullScreen from '../../mixins/full-screen';
  5. import VideoRecord from '../../mixins/video-record';
  6. import ExpandAssets from '../../mixins/expand-assets';
  7. import { audioAssetOptions, imageAssetOptions } from '../../mixins/expand-assets';
  8. import isColor from '../../utils/is-color';
  9.  
  10. let {
  11.     $
  12. } = Ember;
  13.  
  14.  
  15. /**
  16.  * @module exp-player
  17.  * @submodule frames
  18.  */
  19.  
  20. /**
  21.  * Frame to display image(s) and play audio, with optional video recording. Options allow
  22.  * customization for looking time, storybook, forced choice, and reaction time type trials,
  23.  * including training versions where children (or parents) get feedback about their responses.
  24.  *
  25.  * This can be used in a variety of ways - for example:
  26.  *
  27.  * - Display an image for a set amount of time and measure looking time
  28.  *
  29.  * - Display two images for a set amount of time and play audio for a
  30.  * looking-while-listening paradigm
  31.  *
  32.  * - Show a "storybook page" where you show images and play audio, having the parent/child
  33.  * press 'Next' to proceed. If desired,
  34.  * images can appear and be highlighted at specific times
  35.  * relative to audio. E.g., the audio might say "This [image of Remy appears] is a boy
  36.  * named Remy. Remy has a little sister [image of Zenna appears] named Zenna.
  37.  * [Remy highlighted] Remy's favorite food is brussel sprouts, but [Zenna highlighted]
  38.  * Zenna's favorite food is ice cream. [Remy and Zenna both highlighted] Remy and Zenna
  39.  * both love tacos!"
  40.  *
  41.  * - Play audio asking the child to choose between two images by pointing or answering
  42.  * verbally. Show text for the parent about how to help and when to press Next.
  43.  *
  44.  * - Play audio asking the child to choose between two images, and require one of those
  45.  * images to be clicked to proceed (see "choiceRequired" option).
  46.  *
  47.  * - Measure reaction time as the child is asked to choose a particular option on each trial
  48.  * (e.g., a central cue image is shown first, then two options at a short delay; the child
  49.  * clicks on the one that matches the cue in some way)
  50.  *
  51.  * - Provide audio and/or text feedback on the child's (or parent's) choice before proceeding,
  52.  * either just to make the study a bit more interactive ("Great job, you chose the color BLUE!")
  53.  * or for initial training/familiarization to make sure they understand the task. Some
  54.  * images can be marked as the "correct" answer and a correct answer required to proceed.
  55.  * If you'd like to include some initial training questions before your test questions,
  56.  * this is a great way to do it.
  57.  *
  58.  * In general, the images are displayed in a designated region of the screen with aspect
  59.  * ratio 7:4 (1.75 times as wide as it is tall) to standardize display as much as possible
  60.  * across different monitors. If you want to display things truly fullscreen, you can
  61.  * use `autoProceed` and not provide `parentText` so there's nothing at the bottom, and then
  62.  * set `maximizeDisplay` to true.
  63.  *
  64.  * Webcam recording may be turned on or off; if on, stimuli are not displayed and audio is
  65.  * not started until recording begins. (Using the frame-specific `isRecording` property
  66.  * is good if you have a smallish number of test trials and prefer to have separate video
  67.  * clips for each. For reaction time trials or many short trials, you will likely want
  68.  * to use session recording instead - i.e. start the session recording before the first trial
  69.  * and end on the last trial - to avoid the short delays related to starting/stopping the video.)
  70.  *
  71.  * This frame is displayed fullscreen, but is not paused or otherwise disabled if the
  72.  * user leaves fullscreen. A button appears prompting the user to return to
  73.  * fullscreen mode.
  74.  *
  75.  * Any number of images may be placed on the screen, and their position
  76.  * specified. (Aspect ratio will be the same as the original image.)
  77.  *
  78.  * The examples below show a variety of usages, corresponding to those shown in the video.
  79.  *
  80.  * image-1: Single image displayed full-screen, maximizing area on monitor, for 8 seconds.
  81.  *
  82.  * image-2: Single image displayed at specified position, with 'next' button to move on
  83.  *
  84.  * image-3: Image plus audio, auto-proceeding after audio completes and 4 seconds go by
  85.  *
  86.  * image-4: Image plus audio, with 'next' button to move on
  87.  *
  88.  * image-5: Two images plus audio question asking child to point to one of the images,
  89.  *   demonstrating different timing of image display & highlighting of images during audio
  90.  *
  91.  * image-6: Three images with audio prompt, family has to click one of two to continue
  92.  *
  93.  * image-7: Three images with audio prompt, family has to click correct one to continue -
  94.  *   audio feedback on incorrect answer
  95.  *
  96.  * image-8: Three images with audio prompt, family has to click correct one to continue -
  97.  *   text feedback on incorrect answer
  98.  *
  99.  *
  100.  
  101. ```json
  102.  "frames": {
  103.     "image-1": {
  104.         "kind": "exp-lookit-images-audio",
  105.         "images": [
  106.             {
  107.                 "id": "cats",
  108.                 "src": "two_cats.png",
  109.                 "position": "fill"
  110.             }
  111.         ],
  112.         "baseDir": "https://www.mit.edu/~kimscott/placeholderstimuli/",
  113.         "autoProceed": true,
  114.         "doRecording": true,
  115.         "durationSeconds": 8,
  116.         "maximizeDisplay": true
  117.     },
  118.     "image-2": {
  119.         "kind": "exp-lookit-images-audio",
  120.         "images": [
  121.             {
  122.                 "id": "cats",
  123.                 "src": "three_cats.JPG",
  124.                 "top": 10,
  125.                 "left": 30,
  126.                 "width": 40
  127.             }
  128.         ],
  129.         "baseDir": "https://www.mit.edu/~kimscott/placeholderstimuli/",
  130.         "autoProceed": false,
  131.         "doRecording": true,
  132.         "parentTextBlock": {
  133.             "text": "Some explanatory text for parents",
  134.             "title": "For parents"
  135.         }
  136.     },
  137.     "image-3": {
  138.         "kind": "exp-lookit-images-audio",
  139.         "audio": "wheresremy",
  140.         "images": [
  141.             {
  142.                 "id": "remy",
  143.                 "src": "wheres_remy.jpg",
  144.                 "position": "fill"
  145.             }
  146.         ],
  147.         "baseDir": "https://www.mit.edu/~kimscott/placeholderstimuli/",
  148.         "audioTypes": [
  149.             "mp3",
  150.             "ogg"
  151.         ],
  152.         "autoProceed": true,
  153.         "doRecording": false,
  154.         "durationSeconds": 4,
  155.         "parentTextBlock": {
  156.             "text": "Some explanatory text for parents",
  157.             "title": "For parents"
  158.         },
  159.         "showProgressBar": true
  160.     },
  161.     "image-4": {
  162.         "kind": "exp-lookit-images-audio",
  163.         "audio": "peekaboo",
  164.         "images": [
  165.             {
  166.                 "id": "remy",
  167.                 "src": "peekaboo_remy.jpg",
  168.                 "position": "fill"
  169.             }
  170.         ],
  171.         "baseDir": "https://www.mit.edu/~kimscott/placeholderstimuli/",
  172.         "audioTypes": [
  173.             "mp3",
  174.             "ogg"
  175.         ],
  176.         "autoProceed": false,
  177.         "doRecording": false,
  178.         "parentTextBlock": {
  179.             "text": "Some explanatory text for parents",
  180.             "title": "For parents"
  181.         }
  182.     },
  183.     "image-5": {
  184.         "kind": "exp-lookit-images-audio",
  185.         "audio": "remyzennaintro",
  186.         "images": [
  187.             {
  188.                 "id": "remy",
  189.                 "src": "scared_remy.jpg",
  190.                 "position": "left"
  191.             },
  192.             {
  193.                 "id": "zenna",
  194.                 "src": "love_zenna.jpg",
  195.                 "position": "right",
  196.                 "displayDelayMs": 1500
  197.             }
  198.         ],
  199.         "baseDir": "https://www.mit.edu/~kimscott/placeholderstimuli/",
  200.         "highlights": [
  201.             {
  202.                 "range": [
  203.                     0,
  204.                     1.5
  205.                 ],
  206.                 "imageId": "remy"
  207.             },
  208.             {
  209.                 "range": [
  210.                     1.5,
  211.                     3
  212.                 ],
  213.                 "imageId": "zenna"
  214.             }
  215.         ],
  216.         "autoProceed": false,
  217.         "doRecording": true,
  218.         "parentTextBlock": {
  219.             "text": "Some explanatory text for parents",
  220.             "title": "For parents"
  221.         }
  222.     },
  223.     "image-6": {
  224.         "kind": "exp-lookit-images-audio",
  225.         "audio": "matchremy",
  226.         "images": [
  227.             {
  228.                 "id": "cue",
  229.                 "src": "happy_remy.jpg",
  230.                 "position": "center",
  231.                 "nonChoiceOption": true
  232.             },
  233.             {
  234.                 "id": "option1",
  235.                 "src": "happy_zenna.jpg",
  236.                 "position": "left",
  237.                 "displayDelayMs": 2000
  238.             },
  239.             {
  240.                 "id": "option2",
  241.                 "src": "annoyed_zenna.jpg",
  242.                 "position": "right",
  243.                 "displayDelayMs": 2000
  244.             }
  245.         ],
  246.         "baseDir": "https://www.mit.edu/~kimscott/placeholderstimuli/",
  247.         "autoProceed": false,
  248.         "doRecording": true,
  249.         "choiceRequired": true,
  250.         "parentTextBlock": {
  251.             "text": "Some explanatory text for parents",
  252.             "title": "For parents"
  253.         },
  254.         "canMakeChoiceBeforeAudioFinished": true
  255.     },
  256.     "image-7": {
  257.         "kind": "exp-lookit-images-audio",
  258.         "audio": "matchzenna",
  259.         "images": [
  260.             {
  261.                 "id": "cue",
  262.                 "src": "sad_zenna.jpg",
  263.                 "position": "center",
  264.                 "nonChoiceOption": true
  265.             },
  266.             {
  267.                 "id": "option1",
  268.                 "src": "surprised_remy.jpg",
  269.                 "position": "left",
  270.                 "feedbackAudio": "negativefeedback",
  271.                 "displayDelayMs": 3500
  272.             },
  273.             {
  274.                 "id": "option2",
  275.                 "src": "sad_remy.jpg",
  276.                 "correct": true,
  277.                 "position": "right",
  278.                 "displayDelayMs": 3500
  279.             }
  280.         ],
  281.         "baseDir": "https://www.mit.edu/~kimscott/placeholderstimuli/",
  282.         "autoProceed": false,
  283.         "doRecording": true,
  284.         "choiceRequired": true,
  285.         "parentTextBlock": {
  286.             "text": "Some explanatory text for parents",
  287.             "title": "For parents"
  288.         },
  289.         "correctChoiceRequired": true,
  290.         "canMakeChoiceBeforeAudioFinished": false
  291.     },
  292.     "image-8": {
  293.         "kind": "exp-lookit-images-audio",
  294.         "audio": "matchzenna",
  295.         "images": [
  296.             {
  297.                 "id": "cue",
  298.                 "src": "sad_zenna.jpg",
  299.                 "position": "center",
  300.                 "nonChoiceOption": true
  301.             },
  302.             {
  303.                 "id": "option1",
  304.                 "src": "surprised_remy.jpg",
  305.                 "position": "left",
  306.                 "feedbackText": "Try again! Remy looks surprised in that picture. Can you find the picture where he looks sad, like Zenna?",
  307.                 "displayDelayMs": 3500
  308.             },
  309.             {
  310.                 "id": "option2",
  311.                 "src": "sad_remy.jpg",
  312.                 "correct": true,
  313.                 "position": "right",
  314.                 "feedbackText": "Great job! Remy is sad in that picture, just like Zenna is sad.",
  315.                 "displayDelayMs": 3500
  316.             }
  317.         ],
  318.         "baseDir": "https://www.mit.edu/~kimscott/placeholderstimuli/",
  319.         "autoProceed": false,
  320.         "doRecording": true,
  321.         "choiceRequired": true,
  322.         "parentTextBlock": {
  323.             "text": "Some explanatory text for parents",
  324.             "title": "For parents"
  325.         },
  326.         "correctChoiceRequired": true,
  327.         "canMakeChoiceBeforeAudioFinished": false
  328.     }
  329.  }
  330.  
  331.  * ```
  332.  * @class Exp-lookit-images-audio
  333.  * @extends Exp-frame-base
  334.  * @uses Full-screen
  335.  * @uses Video-record
  336.  * @uses Expand-assets
  337.  */
  338.  
  339.  
  340. export default ExpFrameBaseComponent.extend(FullScreen, VideoRecord, ExpandAssets, {
  341.  
  342.     type: 'exp-lookit-images-audio',
  343.     layout: layout,
  344.     displayFullscreen: true, // force fullscreen for all uses of this component
  345.     fullScreenElementId: 'experiment-player', // which element to send fullscreen
  346.     fsButtonID: 'fsButton', // ID of button to go to fullscreen
  347.  
  348.     startedTrial: false, // whether we've started playing audio yet
  349.     _finishing: false, // whether we're currently trying to move to next trial (to prevent overlapping calls)
  350.  
  351.     // Override setting in VideoRecord mixin - only use camera if doing recording
  352.     doUseCamera: Ember.computed.alias('doRecording'),
  353.     startRecordingAutomatically: Ember.computed.alias('doRecording'),
  354.  
  355.     pageTimer: null,
  356.     progressTimer: null,
  357.     nextButtonDisableTimer: null,
  358.     showChoiceTimer: null,
  359.     imageDisplayTimers: null,
  360.  
  361.     autoProceed: false,
  362.     finishedAllAudio: false,
  363.     minDurationAchieved: false,
  364.  
  365.     choiceRequired: false,
  366.     correctChoiceRequired: false,
  367.     correctImageSelected: false,
  368.     canMakeChoice: true,
  369.     showingFeedbackDialog: false,
  370.     selectedImage: null,
  371.  
  372.     audioPlayed: null,
  373.  
  374.     noParentText: false,
  375.  
  376.     assetsToExpand: {
  377.         'audio': ['audio', 'images/feedbackAudio'],
  378.         'video': [],
  379.         'image': ['images/src']
  380.     },
  381.  
  382.     frameSchemaProperties: {
  383.         /**
  384.          * Whether to do webcam recording (will wait for webcam
  385.          * connection before starting audio or showing images if so)
  386.          *
  387.          * @property {Boolean} doRecording
  388.          */
  389.         doRecording: {
  390.             type: 'boolean',
  391.             description: 'Whether to do webcam recording (will wait for webcam connection before starting audio if so'
  392.         },
  393.  
  394.         /**
  395.          * Whether to proceed automatically when all conditions are met, vs. enabling
  396.          * next button at that point. If true: the next, previous, and replay buttons are
  397.          * hidden, and the frame auto-advances after ALL of the following happen
  398.          * (a) the audio segment (if any) completes
  399.          * (b) the durationSeconds (if any) is achieved
  400.          * (c) a choice is made (if required)
  401.          * (d) that choice is correct (if required)
  402.          * (e) the choice audio (if any) completes
  403.          * (f) the choice text (if any) is dismissed
  404.          * If false: the next, previous, and replay buttons (as applicable) are displayed.
  405.          * It becomes possible to press 'next' only once the conditions above are met.
  406.          *
  407.          * @property {Boolean} autoProceed
  408.          * @default false
  409.          */
  410.         autoProceed: {
  411.             type: 'boolean',
  412.             description: 'Whether to proceed automatically after audio (and hide replay/next buttons)',
  413.             default: false
  414.         },
  415.  
  416.         /**
  417.          * Minimum duration of frame in seconds. If set, then it will only
  418.          * be possible to proceed to the next frame after both the audio completes AND
  419.          * this duration is acheived.
  420.          *
  421.          * @property {Number} durationSeconds
  422.          * @default 0
  423.          */
  424.         durationSeconds: {
  425.             type: 'number',
  426.             description: 'Minimum duration of frame in seconds',
  427.             minimum: 0,
  428.             default: 0
  429.         },
  430.  
  431.         /**
  432.          * [Only used if durationSeconds set] Whether to
  433.          * show a progress bar based on durationSeconds in the parent text area.
  434.          *
  435.          * @property {Boolean} showProgressBar
  436.          * @default false
  437.          */
  438.         showProgressBar: {
  439.             type: 'boolean',
  440.             description: 'Whether to show a progress bar based on durationSeconds',
  441.             default: false
  442.         },
  443.  
  444.         /**
  445.          * [Only used if not autoProceed] Whether to
  446.          * show a previous button to allow the participant to go to the previous frame
  447.          *
  448.          * @property {Boolean} showPreviousButton
  449.          * @default true
  450.          */
  451.         showPreviousButton: {
  452.             type: 'boolean',
  453.             default: true,
  454.             description: 'Whether to show a previous button (used only if showing Next button)'
  455.         },
  456.  
  457.         /**
  458.          * [Only used if not autoProceed AND if there is audio] Whether to
  459.          * show a replay button to allow the participant to replay the audio
  460.          *
  461.          * @property {Boolean} showReplayButton
  462.          * @default false
  463.          */
  464.         showReplayButton: {
  465.             type: 'boolean',
  466.             default: true,
  467.             description: 'Whether to show a replay button (used only if showing Next button)'
  468.         },
  469.  
  470.         /**
  471.          * Whether to have the image display area take up the whole screen if possible.
  472.          * This will only apply if (a) there is no parent text and (b) there are no
  473.          * control buttons (next, previous, replay) because the frame auto-proceeds.
  474.          *
  475.          * @property {Boolean} maximizeDisplay
  476.          * @default false
  477.          */
  478.         maximizeDisplay: {
  479.             type: 'boolean',
  480.             default: false,
  481.             description: 'Whether to have the image display area take up the whole screen if possible'
  482.         },
  483.  
  484.         /**
  485.          * Audio file to play at the start of this frame.
  486.          * This can either be an array of {src: 'url', type: 'MIMEtype'} objects, e.g.
  487.          * listing equivalent .mp3 and .ogg files, or can be a single string `filename`
  488.          * which will be expanded based on `baseDir` and `audioTypes` values (see `audioTypes`).
  489.          *
  490.          * @property {Object[]} audio
  491.          * @default []
  492.          *
  493.          */
  494.         audio: {
  495.             anyOf: audioAssetOptions,
  496.             description: 'Audio to play as this frame begins',
  497.             default: []
  498.         },
  499.         /**
  500.          * Text block to display to parent.  (Each field is optional)
  501.          *
  502.          * @property {Object} parentTextBlock
  503.          *   @param {String} title title to display
  504.          *   @param {String} text paragraph of text
  505.          *   @param {Object} css object specifying any css properties
  506.          *      to apply to this section, and their values - e.g.
  507.          *      {'color': 'gray', 'font-size': 'large'}
  508.          */
  509.         parentTextBlock: {
  510.             type: 'object',
  511.             properties: {
  512.                 title: {
  513.                     type: 'string'
  514.                 },
  515.                 text: {
  516.                     type: 'string'
  517.                 },
  518.                 css: {
  519.                     type: 'object',
  520.                     default: {}
  521.                 }
  522.             },
  523.             default: {}
  524.         },
  525.         /**
  526.          * Array of images to display and information about their placement. For each
  527.          * image, you need to specify `src` (image name/URL) and placement (either by
  528.          * providing left/width/top values, or by using a `position` preset).
  529.          *
  530.          * Everything else is optional! This is where you would say that an image should
  531.          * be shown at a delay
  532.          *
  533.          * @property {Object[]} images
  534.          *   @param {String} id unique ID for this image
  535.          *   @param {String} src URL of image source. This can be a full
  536.          *     URL, or relative to baseDir (see baseDir).
  537.          *   @param {String} alt alt-text for image in case it doesn't load and for
  538.          *     screen readers
  539.          *   @param {Number} left left margin, as percentage of story area width. If not provided,
  540.          *     the image is centered horizontally.
  541.          *   @param {Number} width image width, as percentage of story area width. Note:
  542.          *     in general only provide one of width and height; the other will be adjusted to
  543.          *     preserve the image aspect ratio.
  544.          *   @param {Number} top top margin, as percentage of story area height. If not provided,
  545.          *     the image is centered vertically.
  546.          *   @param {Number} height image height, as percentage of story area height. Note:
  547.          *     in general only provide one of width and height; the other will be adjusted to
  548.          *     preserve the image aspect ratio.
  549.          *   @param {String} position one of 'left', 'center', 'right', 'fill' to use presets
  550.          *     that place the image in approximately the left, center, or right third of
  551.          *     the screen or to fill the screen as much as possible.
  552.          *     This overrides left/width/top values if given.
  553.          *   @param {Boolean} nonChoiceOption [Only used if `choiceRequired` is true]
  554.          *     whether this should be treated as a non-clickable option (e.g., this is
  555.          *     a picture of a girl, and the child needs to choose whether the girl has a
  556.          *     DOG or a CAT)
  557.          *   @param {Number} displayDelayMs Delay at which to show the image after trial
  558.          *     start (timing will be relative to any audio or to start of trial if no
  559.          *     audio). Optional; default is to show images immediately.
  560.          *   @param {Object[]} feedbackAudio [Only used if `choiceRequired` is true]
  561.          *      Audio to play upon clicking this image. This can either be an array of
  562.          *     {src: 'url', type: 'MIMEtype'} objects, e.g. listing equivalent .mp3 and
  563.          *     .ogg files, or can be a single string `filename` which will be expanded
  564.          *     based on `baseDir` and `audioTypes` values (see `audioTypes`).
  565.          *   @param {String} feedbackText [Only used if `choiceRequired` is true] Text
  566.          *     to display in a dialogue window upon clicking the image.
  567.          */
  568.         images: {
  569.             type: 'array',
  570.             items: {
  571.                 type: 'object',
  572.                 properties: {
  573.                     'id': {
  574.                         type: 'string'
  575.                     },
  576.                     'src': {
  577.                         anyOf: imageAssetOptions
  578.                     },
  579.                     'alt': {
  580.                         type: 'string'
  581.                     },
  582.                     'left': {
  583.                         type: 'number'
  584.                     },
  585.                     'width': {
  586.                         type: 'number'
  587.                     },
  588.                     'top': {
  589.                         type: 'number'
  590.                     },
  591.                     'height': {
  592.                         type: 'number'
  593.                     },
  594.                     'position': {
  595.                         type: 'string',
  596.                         enum: ['left', 'center', 'right', 'fill']
  597.                     },
  598.                     'nonChoiceOption': {
  599.                         type: 'boolean'
  600.                     },
  601.                     'displayDelayMs': {
  602.                         type: 'number',
  603.                         minimum: 0
  604.                     },
  605.                     'correct': {
  606.                         type: 'boolean'
  607.                     },
  608.                     'feedbackAudio': {
  609.                         anyOf: audioAssetOptions
  610.                     },
  611.                     'feedbackText': {
  612.                         type: 'string'
  613.                     }
  614.                 }
  615.             }
  616.         },
  617.         /**
  618.          * Color of background. See https://developer.mozilla.org/en-US/docs/Web/CSS/color_value
  619.          * for acceptable syntax: can use color names ('blue', 'red', 'green', etc.), or
  620.          * rgb hex values (e.g. '#800080' - include the '#')
  621.          *
  622.          * @property {String} backgroundColor
  623.          * @default 'black'
  624.          */
  625.         backgroundColor: {
  626.             type: 'string',
  627.             description: 'Color of background',
  628.             default: 'black'
  629.         },
  630.         /**
  631.          * Color of area where images are shown, if different from overall background.
  632.          * Defaults to backgroundColor if one is provided. See
  633.          * https://developer.mozilla.org/en-US/docs/Web/CSS/color_value
  634.          * for acceptable syntax: can use color names ('blue', 'red', 'green', etc.), or
  635.          * rgb hex values (e.g. '#800080' - include the '#')
  636.          *
  637.          * @property {String} pageColor
  638.          * @default 'white'
  639.          */
  640.         pageColor: {
  641.             type: 'string',
  642.             description: 'Color of image area',
  643.             default: 'white'
  644.         },
  645.         /**
  646.          * Whether this is a frame where the user needs to click to select one of the
  647.          * images before proceeding.
  648.          *
  649.          * @property {Boolean} choiceRequired
  650.          * @default false
  651.          */
  652.         choiceRequired: {
  653.             type: 'boolean',
  654.             description: 'Whether this is a frame where the user needs to click to select one of the images before proceeding',
  655.             default: false
  656.         },
  657.         /**
  658.          * [Only used if `choiceRequired` is true] Whether the participant has to select
  659.          * one of the *correct* images before proceeding.
  660.          *
  661.          * @property {Boolean} correctChoiceRequired
  662.          * @default false
  663.          */
  664.         correctChoiceRequired: {
  665.             type: 'boolean',
  666.             description: 'Whether this is a frame where the user needs to click a correct image before proceeding',
  667.             default: false
  668.         },
  669.         /**
  670.          * Whether the participant can make a choice before audio finishes. (Only relevant
  671.          * if `choiceRequired` is true.)
  672.          *
  673.          * @property {Boolean} canMakeChoiceBeforeAudioFinished
  674.          * @default false
  675.          */
  676.         canMakeChoiceBeforeAudioFinished: {
  677.             type: 'boolean',
  678.             description: 'Whether the participant can select an option before audio finishes',
  679.             default: false
  680.         },
  681.         /**
  682.          * Array representing times when particular images should be highlighted. Each
  683.          * element of the array should be of the form {'range': [3.64, 7.83], 'imageId': 'myImageId'}.
  684.          * The two `range` values are the start and end times of the highlight in seconds,
  685.          * relative to the audio played. The `imageId` corresponds to the `id` of an
  686.          * element of `images`.
  687.          *
  688.          * Highlights can overlap in time. Any that go longer than the audio will just
  689.          * be ignored/cut off.
  690.          *
  691.          * One strategy for generating a bunch of highlights for a longer story is to
  692.          * annotate using Audacity and export the labels to get the range values.
  693.          *
  694.          * @property {Object[]} highlights
  695.          *   @param {Array} range [startTimeInSeconds, endTimeInSeconds], e.g. [3.64, 7.83]
  696.          *   @param {String} imageId ID of the image to highlight, corresponding to the `id` field of the element of `images` to highlight
  697.          */
  698.         highlights: {
  699.             type: 'array',
  700.             items: {
  701.                 type: 'object',
  702.                 properties: {
  703.                     'range': {
  704.                         type: 'array',
  705.                         items: {
  706.                             type: 'number',
  707.                             minimum: 0
  708.                         }
  709.                     },
  710.                     'imageId': {
  711.                         'type': 'string'
  712.                     }
  713.                 }
  714.             },
  715.             default: []
  716.         }
  717.     },
  718.  
  719.     meta: {
  720.         data: {
  721.             type: 'object',
  722.             properties: {
  723.                 videoId: {
  724.                     type: 'string'
  725.                 },
  726.                 videoList: {
  727.                     type: 'list'
  728.                 },
  729.                 /**
  730.                 * Array of images used in this frame [same as passed to this frame, but
  731.                 * may reflect random assignment for this particular participant]
  732.                 * @attribute {Array} images
  733.                 */
  734.                 images: {
  735.                     type: 'array'
  736.                 },
  737.                 /**
  738.                 * ID of image selected at time of proceeding
  739.                 * @attribute {String} selectedImage
  740.                 */
  741.                 selectedImage: {
  742.                     type: 'string'
  743.                 },
  744.                 /**
  745.                 * Whether image selected at time of proceeding is marked as correct
  746.                 * @attribute {Boolean} correctImageSelected
  747.                 */
  748.                 correctImageSelected: {
  749.                     type: 'Boolean'
  750.                 },
  751.                 /**
  752.                 * Source URL of audio played, if any. If multiple sources provided (e.g.
  753.                 * mp4 and ogg versions) just the first is stored.
  754.                 * @attribute {String} audioPlayed
  755.                 */
  756.                 audioPlayed: {
  757.                     type: 'string'
  758.                 }
  759.             },
  760.         }
  761.     },
  762.  
  763.     // Override to do a bit extra when starting recording
  764.     onRecordingStarted() {
  765.         this.startTrial();
  766.     },
  767.  
  768.     // Override to do a bit extra when starting session recorder
  769.     onSessionRecordingStarted() {
  770.         this.startTrial();
  771.         $('#waitForVideo').hide();
  772.     },
  773.  
  774.     updateCharacterHighlighting() {
  775.  
  776.         var highlights = this.get('highlights');
  777.  
  778.         if (highlights.length) {
  779.             var t = $('#player-audio')[0].currentTime;
  780.             $('.story-image-container img.story-image').removeClass('narration-highlight');
  781.             // var _this = this;
  782.             highlights.forEach(function (h) {
  783.                 if (t > h.range[0] && t < h.range[1]) {
  784.                     var $element = $('#' + h.imageId + ' img.story-image')
  785.                     $element.addClass('narration-highlight');
  786.                     // _this.wiggle($element);
  787.                 }
  788.             });
  789.         }
  790.     },
  791.  
  792.     // Move an image up and down until the isSpeaking class is removed.
  793.     // Yes, this could much more naturally be done by using a CSS animation property
  794.     // on isSpeaking, but despite animations getting applied properly to the element,
  795.     // I haven't been able to get that working - because of the possibility of ember-
  796.     // specific problems here, I'm going with something that *works* even if it's less
  797.     // elegent.
  798.     //     wiggle($element) {
  799.     //         var _this = this;
  800.     //         var $parent = $element.parent();
  801.     //         if ($element.hasClass('narration-highlight')) {
  802.     //             $parent.animate({'margin-bottom': '.1%', 'margin-top': '-.1%'}, 150, function() {
  803.     //                 if ($element.hasClass('narration-highlight')) {
  804.     //                     $parent.animate({'margin-bottom': '0%', 'margin-top': '0%'}, 150, function() {
  805.     //                         _this.wiggle($element);
  806.     //                     });
  807.     //                 }
  808.     //             });
  809.     //         }
  810.     //     },
  811.  
  812.     replay() {
  813.         // pause any current audio, and set times to 0
  814.         $('audio').each(function() {
  815.             this.pause();
  816.             this.currentTime = 0;
  817.         });
  818.         /**
  819.          * When main audio segment is replayed
  820.          *
  821.          * @event replayAudio
  822.          */
  823.         this.send('setTimeEvent', 'replayAudio');
  824.         // restart audio
  825.         $(`.story-image-container`).hide();
  826.         this.showImages();
  827.         this.playAudio();
  828.     },
  829.  
  830.     finish() {
  831.         if (!this.get('_finishing')) {
  832.             this.set('_finishing', true);
  833.             var _this = this;
  834.             /**
  835.              * Trial is complete and attempting to move to next frame; may wait for recording
  836.              * to catch up before proceeding.
  837.              *
  838.              * @event trialComplete
  839.              */
  840.             this.send('setTimeEvent', 'trialComplete');
  841.             if (this.get('doRecording')) {
  842.                 $('#nextbutton').text('Sending recording...');
  843.                 $('#nextbutton').prop('disabled', true);
  844.                 this.set('nextButtonDisableTimer', window.setTimeout(function () {
  845.                     $('#nextbutton').prop('disabled', false);
  846.                 }, 5000));
  847.  
  848.                 this.stopRecorder().then(() => {
  849.                     _this.set('stoppedRecording', true);
  850.                     _this.send('next');
  851.                 }, () => {
  852.                     _this.send('next');
  853.                 });
  854.             } else {
  855.                 _this.send('next');
  856.             }
  857.         }
  858.     },
  859.  
  860.     finishedAudio() {
  861.         /**
  862.          * When main audio segment finishes playing
  863.          *
  864.          * @event finishAudio
  865.          */
  866.         this.send('setTimeEvent', 'finishAudio');
  867.         this.set('finishedAllAudio', true);
  868.         this.set('canMakeChoice', true);
  869.         this.checkAndEnableProceed();
  870.     },
  871.  
  872.     checkAndEnableProceed() {
  873.         let ready = this.get('minDurationAchieved') &&
  874.                     this.get('finishedAllAudio') &&
  875.                     !this.get('showingFeedbackDialog') &&
  876.                     (!this.get('choiceRequired') || (this.get('selectedImage') && (this.get('correctImageSelected') || !this.get('correctChoiceRequired'))));
  877.         if (ready) {
  878.             this.readyToFinish();
  879.         }
  880.         return ready;
  881.     },
  882.  
  883.     readyToFinish() {
  884.         if (this.get('autoProceed')) {
  885.             this.send('finish');
  886.         } else {
  887.             $('#nextbutton').prop('disabled', false);
  888.         }
  889.     },
  890.  
  891.     startTrial() {
  892.         this.set('startedTrial', true);
  893.         if (this.get('durationSeconds') && this.get('durationSeconds') > 0) {
  894.             let _this = this;
  895.             /**
  896.             * Timer for set-duration trial begins
  897.             *
  898.             * @event startTimer
  899.             */
  900.             this.send('setTimeEvent', 'startTimer');
  901.             this.set('pageTimer', window.setTimeout(function() {
  902.                 /**
  903.                     * Timer for set-duration trial ends
  904.                     *
  905.                     * @event endTimer
  906.                     */
  907.                 _this.send('setTimeEvent', 'endTimer');
  908.                 _this.set('minDurationAchieved', true);
  909.                 _this.checkAndEnableProceed();
  910.             }, _this.get('durationSeconds') * 1000));
  911.             if (this.get('showProgressBar')) {
  912.                 let timerStart = new Date().getTime();
  913.                 let durationSeconds = _this.get('durationSeconds');
  914.                 this.set('progressTimer', window.setInterval(function() {
  915.                     let now = new Date().getTime();
  916.                     var prctDone =  (now - timerStart) / (durationSeconds * 10);
  917.                     $('.progress-bar').css('width', prctDone + '%');
  918.                 }, 100));
  919.             }
  920.         } else {
  921.             this.set('minDurationAchieved', true);
  922.         }
  923.  
  924.         this.playAudio();
  925.         this.showImages();
  926.     },
  927.  
  928.     playAudio() {
  929.         // Start audio if there is any
  930.         var _this = this;
  931.         if ($('#player-audio source').length) {
  932.             $('#player-audio')[0].play().then(() => {
  933.                 /**
  934.                  * When main audio segment starts playing
  935.                  *
  936.                  * @event startAudio
  937.                  */
  938.                 _this.send('setTimeEvent', 'startAudio');
  939.             }, () => {
  940.                 /**
  941.                  * When main audio cannot be started. In this case we treat it as if
  942.                  * the audio was completed (for purposes of allowing participant to
  943.                  * proceed)
  944.                  *
  945.                  * @event failedToStartAudio
  946.                  */
  947.                 _this.send('setTimeEvent', 'failedToStartAudio');
  948.                 _this.finishedAudio();
  949.             });
  950.         } else { // Otherwise treat as if completed
  951.             this.finishedAudio();
  952.         }
  953.     },
  954.  
  955.     showImages() {
  956.         /**
  957.          * When images are displayed to participant (for images without any delay added)
  958.          *
  959.          * @event displayAllImages
  960.          */
  961.         this.send('setTimeEvent', 'displayImages');
  962.         var _this = this;
  963.         $.each(this.get('images_parsed'), function(idx, image) {
  964.             if (image.hasOwnProperty('displayDelayMs')) {
  965.                 var thisTimeout = window.setTimeout(function() {
  966.                     /**
  967.                      * When a specific image is shown at a delay.
  968.                      *
  969.                      * @event displayImage
  970.                      * @param {String} imageId
  971.                      */
  972.                     _this.send('setTimeEvent', 'displayImage', {
  973.                         imageId: image.id
  974.                     });
  975.                     $(`.story-image-container#${image.id}`).show();
  976.                 }, image.displayDelayMs);
  977.                 if (_this.get('imageDisplayTimers')) {
  978.                     _this.get('imageDisplayTimers').push(thisTimeout);
  979.                 } else {
  980.                     _this.set('imageDisplayTimers', [thisTimeout]);
  981.                 }
  982.             } else {
  983.                 $(`.story-image-container#${image.id}`).show();
  984.             }
  985.         });
  986.     },
  987.  
  988.     clickImage(imageId, nonChoiceOption, correct, feedbackText) {
  989.         // If this is a choice frame and a valid choice and we're allowed to make a choice yet...
  990.         if (this.get('choiceRequired') && !nonChoiceOption && this.get('canMakeChoice') && !this.get('showingFeedbackDialog')) {
  991.             this.set('finishedAllAudio', true); // Treat as if audio is finished in case making choice before audio finishes - otherwise we never satisfy that criterion
  992.             /**
  993.              * When one of the image options is clicked during a choice frame
  994.              *
  995.              * @event clickImage
  996.              * @param {String} imageId ID of the image selected
  997.              * @param {Boolean} correct whether this image is marked as correct
  998.              */
  999.             this.send('setTimeEvent', 'clickImage', {
  1000.                 imageId: imageId,
  1001.                 correct: correct
  1002.             });
  1003.  
  1004.             // Highlight the selected image and store it
  1005.             $('.story-image-container img').removeClass('highlight');
  1006.             $('#' + imageId + ' img').addClass('highlight');
  1007.             this.set('selectedImage', imageId);
  1008.             this.set('correctImageSelected', correct);
  1009.  
  1010.             if (this.get('correctChoiceRequired') && !correct) {
  1011.                 $('#nextbutton').prop('disabled', true);
  1012.             }
  1013.  
  1014.             var noFeedback = true; // Track whether we're giving some form of feedback
  1015.             // vs. allowing immediate proceeding
  1016.  
  1017.             var _this = this;
  1018.             // Play any feedback audio if available
  1019.             if ($(`#${imageId}.story-image-container audio source`).length) {
  1020.                 noFeedback = false;
  1021.                 // If there's audio associated with this choice,
  1022.                 $('audio').each(function() { // pause any other audio
  1023.                     this.pause();
  1024.                     this.currentTime = 0;
  1025.                 });
  1026.                 $(`#${imageId}.story-image-container audio`)[0].play().then(() => {
  1027.                     /**
  1028.                      * When image/feedback audio is started
  1029.                      *
  1030.                      * @event startImageAudio
  1031.                      * @param {String} imageId
  1032.                      */
  1033.                     _this.send('setTimeEvent', 'startImageAudio', {
  1034.                         imageId: imageId
  1035.                     });
  1036.                 }, () => {
  1037.                     /**
  1038.                      * When image/feedback audio cannot be started. In this case we treat it as if
  1039.                      * the audio was completed (for purposes of allowing participant to
  1040.                      * proceed)
  1041.                      *
  1042.                      * @event failedToStartImageAudio
  1043.                      * @param {String} imageId
  1044.                      */
  1045.                     _this.send('setTimeEvent', 'failedToStartImageAudio', {
  1046.                         imageId: imageId
  1047.                     });
  1048.                     _this.endFeedbackAudio(imageId, correct);
  1049.                 });
  1050.             }
  1051.  
  1052.             // Also display any feedback text if available
  1053.             if (feedbackText) {
  1054.                 noFeedback = false;
  1055.                 this.set('showingFeedbackDialog', true);
  1056.                 $(`.${imageId}.modal`).show();
  1057.             }
  1058.  
  1059.             // If we're giving feedback (audio or text), it will be possible to proceed
  1060.             // once any audio finishes and any text is dismissed. Otherwise, just ensure
  1061.             // that if we're moving on, the answer gets highlighted long enough for
  1062.             // the participant to see it!
  1063.             if (noFeedback) {
  1064.                 this.set('showChoiceTimer', window.setTimeout(function() {
  1065.                     window.clearInterval(_this.get('showChoiceTimer'));
  1066.                     _this.checkAndEnableProceed();
  1067.                 }, 150));
  1068.             }
  1069.  
  1070.         }
  1071.     },
  1072.  
  1073.     endFeedbackAudio(imageId, correct) {  // eslint-disable-line no-unused-vars
  1074.         this.checkAndEnableProceed(); // if correct, move on
  1075.     },
  1076.  
  1077.     actions: {
  1078.  
  1079.         // During playing audio
  1080.         updateCharacterHighlighting() {
  1081.             this.updateCharacterHighlighting();
  1082.         },
  1083.  
  1084.         replay() {
  1085.             this.replay();
  1086.         },
  1087.  
  1088.         finish() {
  1089.             this.finish();
  1090.         },
  1091.  
  1092.         finishedAudio() {
  1093.             this.finishedAudio();
  1094.         },
  1095.  
  1096.         clickImage(imageId, nonChoiceOption, correct, feedbackText) {
  1097.             this.clickImage(imageId, nonChoiceOption, correct, feedbackText);
  1098.         },
  1099.  
  1100.         endFeedbackAudio(imageId, correct) {
  1101.             this.endFeedbackAudio(imageId, correct);
  1102.         },
  1103.  
  1104.         hideFeedbackDialog(imageId) {
  1105.             $(`.${imageId}.modal`).hide();
  1106.             this.set('showingFeedbackDialog', false);
  1107.             /**
  1108.              * When the participant dismisses a feedback dialogue
  1109.              *
  1110.              * @event dismissFeedback
  1111.              * @param {String} imageId
  1112.              */
  1113.             this.send('setTimeEvent', 'dismissFeedback', {
  1114.                 imageId: imageId
  1115.             });
  1116.             this.checkAndEnableProceed();
  1117.         }
  1118.     },
  1119.  
  1120.     // Supply image IDs if they're missing.
  1121.     didReceiveAttrs() {
  1122.         this._super(...arguments);
  1123.         var N = 1;
  1124.         var allImageIds =  this.get('images') ? this.get('images').map(im => im.hasOwnProperty('id') ? im.id : null) : [];
  1125.         $.each(this.get('images'), function(idx, image) {
  1126.             if (!image.hasOwnProperty('id')) {
  1127.                 while (allImageIds.includes(`image_${N}`)) {
  1128.                     N++;
  1129.                 }
  1130.                 image.id = `image_${N}`;
  1131.             }
  1132.             N++;
  1133.         });
  1134.  
  1135.         this.set('canMakeChoice', !!this.get('canMakeChoiceBeforeAudioFinished'));
  1136.         this.set('minDurationAchieved', !(this.get('durationSeconds') > 0));
  1137.         this.set('showProgressBar', this.get('showProgressBar') && this.get('durationSeconds') > 0);
  1138.         this.set('showReplayButton', this.get('showReplayButton') && this.get('audio').length);
  1139.  
  1140.         // Store audio source
  1141.         let audioSources = this.get('audio_parsed');
  1142.         if (audioSources && audioSources.length) {
  1143.             this.set('audioPlayed', audioSources[0].src);
  1144.         }
  1145.     },
  1146.  
  1147.     didInsertElement() {
  1148.  
  1149.         this._super(...arguments);
  1150.  
  1151.         // Apply user-provided CSS to parent text block
  1152.         if (Object.keys(this.get('parentTextBlock')).length) {
  1153.             var parentTextBlock = this.get('parentTextBlock') || {};
  1154.             var css = parentTextBlock.css || {};
  1155.             $('#parenttext').css(css);
  1156.         } else {
  1157.             this.set('noParentText', true);
  1158.             if (this.get('autoProceed')) {
  1159.                 this.set('noStoryControls', true);
  1160.             }
  1161.         }
  1162.  
  1163.         // Apply user-provided CSS to images
  1164.         $.each(this.get('images_parsed'), function(idx, image) {
  1165.             if (!image.position) {
  1166.                 $('#' + image.id).css({'left': `${image.left}%`, 'width': `${image.width}%`, 'top': `${image.top}%`, 'height': `${image.height}%`});
  1167.             }
  1168.         });
  1169.  
  1170.         // Apply background colors
  1171.         if (isColor(this.get('backgroundColor'))) {
  1172.             $('div.exp-lookit-image-audio').css('background-color', this.get('backgroundColor'));
  1173.         } else {
  1174.             console.warn('Invalid background color provided; not applying.');
  1175.         }
  1176.  
  1177.         if (isColor(this.get('pageColor'))) {
  1178.             $('div.exp-lookit-image-audio div#image-area').css('background-color', this.get('pageColor'));
  1179.         } else {
  1180.             console.warn('Invalid page color provided; not applying.');
  1181.         }
  1182.  
  1183.         $('#nextbutton').prop('disabled', true);
  1184.  
  1185.         // Begin trial!
  1186.         this.checkAndEnableProceed();
  1187.     },
  1188.  
  1189.     // Once rendered, hide images and (if not recording) begin trial
  1190.     didRender() {
  1191.         if (!this.get('startedTrial')) { // don't re-hide/re-start upon e.g. rerender
  1192.             $('.story-image-container').hide();
  1193.             // If recording, trial will be started upon recording start. Otherwise...
  1194.             if (!this.get('doRecording') && !this.get('startSessionRecording')) {
  1195.                 this.startTrial();
  1196.             }
  1197.         }
  1198.     },
  1199.  
  1200.     willDestroyElement() {
  1201.         // Clear any timers that might be active
  1202.         window.clearInterval(this.get('pageTimer'));
  1203.         window.clearInterval(this.get('progressTimer'));
  1204.         window.clearInterval(this.get('nextButtonDisableTimer'));
  1205.         window.clearInterval(this.get('showChoiceTimer'));
  1206.         $.each(this.get('imageDisplayTimers'), function(idx, timeout) {
  1207.             window.clearInterval(timeout);
  1208.         });
  1209.  
  1210.         this._super(...arguments);
  1211.     },
  1212.  
  1213.  
  1214. });
  1215.  
  1216.