diff --git a/app/controllers/course/admin/assessment_settings_controller.rb b/app/controllers/course/admin/assessment_settings_controller.rb index 2b9a8947188..30229c9e745 100644 --- a/app/controllers/course/admin/assessment_settings_controller.rb +++ b/app/controllers/course/admin/assessment_settings_controller.rb @@ -69,6 +69,8 @@ def category_params # :allow_randomization, :allow_mrq_options_randomization, :programming_max_time_limit, + :rubric_grading_prompt_enabled, + :rubric_grading_prompt, assessment_categories_attributes: [ :id, :title, diff --git a/app/models/concerns/course/assessment/submission/answers_concern.rb b/app/models/concerns/course/assessment/submission/answers_concern.rb index 1e65b440d48..6aea3d6163b 100644 --- a/app/models/concerns/course/assessment/submission/answers_concern.rb +++ b/app/models/concerns/course/assessment/submission/answers_concern.rb @@ -52,9 +52,6 @@ def bulk_save_new_answer_actables(new_answers_group_by_actables) ActiveRecord::Base.transaction do new_answers_group_by_actables.each_key do |key| key.constantize.import! new_answers_group_by_actables[key], recursive: true - if key.constantize == Course::Assessment::Answer::RubricBasedResponse - new_answers_group_by_actables[key].each(&:create_category_grade_instances) - end end end end diff --git a/app/models/course.rb b/app/models/course.rb index e428ef5d993..eb1b8ee0edd 100644 --- a/app/models/course.rb +++ b/app/models/course.rb @@ -262,6 +262,27 @@ def programming_max_time_limit=(time) settings(:course_assessments_component).programming_max_time_limit = time end + # Whether the course-wide grading prompt is applied during rubric AI grading. Independent of the prompt + # text so toggling off preserves the text for later re-enabling. + def rubric_grading_prompt_enabled + settings(:course_assessments_component).rubric_grading_prompt_enabled || false + end + + def rubric_grading_prompt_enabled=(enabled) + settings(:course_assessments_component).rubric_grading_prompt_enabled = + ActiveRecord::Type::Boolean.new.cast(enabled) + end + + # Course-wide prompt prepended before each question's own grading prompt during rubric AI grading, applied + # only when rubric_grading_prompt_enabled (see Course::Rubric::RubricAdapter#grading_prompt). + def rubric_grading_prompt + settings(:course_assessments_component).rubric_grading_prompt + end + + def rubric_grading_prompt=(prompt) + settings(:course_assessments_component).rubric_grading_prompt = prompt.presence + end + def codaveri_feedback_workflow settings(:course_codaveri_component).feedback_workflow end diff --git a/app/models/course/assessment/answer/rubric_based_response.rb b/app/models/course/assessment/answer/rubric_based_response.rb index be8abf26f8a..dfe4074003f 100644 --- a/app/models/course/assessment/answer/rubric_based_response.rb +++ b/app/models/course/assessment/answer/rubric_based_response.rb @@ -8,11 +8,11 @@ class Course::Assessment::Answer::RubricBasedResponse < ApplicationRecord # record, so they are stashed during #assign_params and persisted once the answer itself saves. after_save :persist_grading_selections, if: -> { @pending_grading_selections.present? } + # Deprecated link to the v1 response selection model. Remains because v1 tables will not be removed. + # Replaced by an indirect link via Answer -> Rubric::AnswerEvaluation -> Rubric::AnswerEvaluation::Selection has_many :selections, class_name: 'Course::Assessment::Answer::RubricBasedResponseSelection', dependent: :destroy, foreign_key: :answer_id, inverse_of: :answer - accepts_nested_attributes_for :selections, allow_destroy: true - # Specific implementation of Course::Assessment::Answer#reset_answer def reset_answer self.answer_text = question.actable.template_text || '' @@ -86,23 +86,6 @@ def ensure_grading_evaluation! end end - def create_category_grade_instances - answer.class.transaction do - new_category_selections = question.specific.categories.map do |category| - { - answer_id: id, - category_id: category.id, - criterion_id: nil, - grade: nil, - explanation: nil - } - end - - selections = Course::Assessment::Answer::RubricBasedResponseSelection.insert_all(new_category_selections) - raise ActiveRecord::Rollback if !new_category_selections.empty? && (selections.nil? || selections.rows.empty?) - end - end - private def set_default diff --git a/app/models/course/assessment/question.rb b/app/models/course/assessment/question.rb index 377cd70b71c..24502a77fae 100644 --- a/app/models/course/assessment/question.rb +++ b/app/models/course/assessment/question.rb @@ -201,8 +201,10 @@ def copy_attributes(other) end # Duplicates this question's grading contexts (see Course::Assessment::Question::GradingContext). Called from - # each rubric-gradable actable's #initialize_duplicate with +other+ = the source actable being duplicated. - # * As a CONSUMER: deep-copy the contexts this question pulls from onto the duplicate. + # the #initialize_duplicate of every type that can consume contexts (RBR, forum) OR be a sibling-answer + # source (text response), with +other+ = the source actable being duplicated. + # * As a CONSUMER: deep-copy the contexts this question pulls from onto the duplicate (no-op for source-only + # types, which have none). # * As a SOURCE: re-point any already-duplicated contexts that reference this question, so a duplicate # consumer pulls from the duplicate source. Together with GradingContext#initialize_duplicate, the source # linkage is preserved regardless of which of the two questions is duplicated first. diff --git a/app/models/course/assessment/question/forum_post_response.rb b/app/models/course/assessment/question/forum_post_response.rb index 66dcb9f6a5c..815ea261337 100644 --- a/app/models/course/assessment/question/forum_post_response.rb +++ b/app/models/course/assessment/question/forum_post_response.rb @@ -59,6 +59,12 @@ def attempt(submission, last_attempt = nil) def initialize_duplicate(duplicator, other) copy_attributes(other) + + # active_rubric lives on the polymorphic question; the dup'd acting_as carries the source's + # active_rubric_id over, so replace it with a duplicate of the source rubric (re-homed to the destination + # course by Course::Rubric#initialize_duplicate) instead of sharing the source's. Only rubric-mode forum + # questions have one; a default-mode question leaves it nil. + self.active_rubric = duplicator.duplicate(other.active_rubric) if other.active_rubric initialize_grading_context_duplicates(duplicator, other) end diff --git a/app/models/course/assessment/question/text_response.rb b/app/models/course/assessment/question/text_response.rb index be369b00aea..9e79611460d 100644 --- a/app/models/course/assessment/question/text_response.rb +++ b/app/models/course/assessment/question/text_response.rb @@ -146,6 +146,10 @@ def initialize_duplicate(duplicator, other) else self.solutions = duplicator.duplicate(other.solutions) end + + # A text response is never a context CONSUMER, but it can be a sibling-answer SOURCE, so it still needs + # the source fix-up pass when duplicated (see Course::Assessment::Question#initialize_grading_context_duplicates). + initialize_grading_context_duplicates(duplicator, other) end def build_at_least_one_group_one_point diff --git a/app/models/course/rubric/rubric_adapter.rb b/app/models/course/rubric/rubric_adapter.rb index 7526e2fdfd9..2ab91eba7f4 100644 --- a/app/models/course/rubric/rubric_adapter.rb +++ b/app/models/course/rubric/rubric_adapter.rb @@ -19,8 +19,11 @@ def formatted_rubric_categories end.join("\n\n") end + # The course-wide grading prompt (set in the assessment settings page) is prepended before the question's + # own grading prompt when enabled and present, so shared grading guidance applies to every rubric-graded + # question in the course. Either part may be blank; the separator only appears when both are present. def grading_prompt - @rubric.grading_prompt + [course_grading_prompt, @rubric.grading_prompt.presence].compact.join("\n\n") end def model_answer @@ -62,4 +65,13 @@ def build_category_schema(category, field_name) 'description' => "Selected criterion and explanation for #{field_name} #{category.name}" } end + + private + + def course_grading_prompt + course = @rubric.course + return nil unless course.rubric_grading_prompt_enabled + + course.rubric_grading_prompt.presence + end end diff --git a/app/views/course/admin/assessment_settings/edit.json.jbuilder b/app/views/course/admin/assessment_settings/edit.json.jbuilder index ee89c76d0f5..a4ab1558686 100644 --- a/app/views/course/admin/assessment_settings/edit.json.jbuilder +++ b/app/views/course/admin/assessment_settings/edit.json.jbuilder @@ -6,6 +6,8 @@ json.showStdoutAndStderr current_course.show_stdout_and_stderr || false json.allowRandomization current_course.allow_randomization || false json.allowMrqOptionsRandomization current_course.allow_mrq_options_randomization || false json.maxProgrammingTimeLimit current_course.programming_max_time_limit if can?(:manage, :all) +json.rubricGradingPromptEnabled current_course.rubric_grading_prompt_enabled +json.rubricGradingPrompt current_course.rubric_grading_prompt || '' json.canCreateCategories can?(:create, Course::Assessment::Category.new(course: current_course)) diff --git a/client/app/bundles/course/admin/pages/AssessmentSettings/AssessmentSettingsForm.tsx b/client/app/bundles/course/admin/pages/AssessmentSettings/AssessmentSettingsForm.tsx index 7394731f87a..f1451855227 100644 --- a/client/app/bundles/course/admin/pages/AssessmentSettings/AssessmentSettingsForm.tsx +++ b/client/app/bundles/course/admin/pages/AssessmentSettings/AssessmentSettingsForm.tsx @@ -31,6 +31,13 @@ const AssessmentsSettingsForm = forwardRef< .nullable() .typeError(t(translations.maxTimeLimitRequired)) .min(1, t(translations.positiveMaxTimeLimitRequired)), + rubricGradingPrompt: yup.string().when('rubricGradingPromptEnabled', { + is: true, + then: yup + .string() + .trim() + .required(t(translations.rubricGradingPromptRequired)), + }), }); return ( @@ -42,7 +49,7 @@ const AssessmentsSettingsForm = forwardRef< onSubmit={props.onSubmit} validates={validationSchema} > - {(control): JSX.Element => ( + {(control, watch): JSX.Element => ( <>
{/* Randomized Assessment is temporarily hidden (PR#5406) */} @@ -137,6 +144,49 @@ const AssessmentsSettingsForm = forwardRef< )}
+
+ ( + + )} + /> + + + {t(translations.rubricGradingPromptHint)} + + + ( + + )} + /> +
+
({ id: category.id, title: category.title, diff --git a/client/app/bundles/course/admin/pages/AssessmentSettings/translations.ts b/client/app/bundles/course/admin/pages/AssessmentSettings/translations.ts index bcab1d77a72..d5f57ae555a 100644 --- a/client/app/bundles/course/admin/pages/AssessmentSettings/translations.ts +++ b/client/app/bundles/course/admin/pages/AssessmentSettings/translations.ts @@ -58,6 +58,24 @@ export default defineMessages({ id: 'course.admin.AssessmentSettings.enableMcqChoicesRandomisations', defaultMessage: 'Randomise MCQ choices', }, + rubricGrading: { + id: 'course.admin.AssessmentSettings.rubricGrading', + defaultMessage: 'AI Rubric Grading', + }, + useRubricGradingPrompt: { + id: 'course.admin.AssessmentSettings.useRubricGradingPrompt', + defaultMessage: 'Use course-wide grading prompt', + }, + rubricGradingPromptHint: { + id: 'course.admin.AssessmentSettings.rubricGradingPromptHint', + defaultMessage: + "When a question with a grading rubric is autograded with AI, these instructions will be inserted before the question's specific grading prompt.", + }, + rubricGradingPromptRequired: { + id: 'course.admin.AssessmentSettings.rubricGradingPromptRequired', + defaultMessage: + 'Please enter a grading prompt, or disable the course-wide grading prompt.', + }, deleteCategoryPromptAction: { id: 'course.admin.AssessmentSettings.deleteCategoryPromptAction', defaultMessage: 'Delete {title} category', diff --git a/client/app/types/course/admin/assessments.ts b/client/app/types/course/admin/assessments.ts index 61655d43484..ba7c35e8b44 100644 --- a/client/app/types/course/admin/assessments.ts +++ b/client/app/types/course/admin/assessments.ts @@ -6,6 +6,8 @@ export interface AssessmentSettingsData { categories: AssessmentCategory[]; canCreateCategories: boolean; maxProgrammingTimeLimit?: number; + rubricGradingPromptEnabled: boolean; + rubricGradingPrompt?: string; } export interface AssessmentCategory { @@ -62,6 +64,8 @@ export interface AssessmentSettingsPostData { allow_randomization?: AssessmentSettingsData['allowRandomization']; allow_mrq_options_randomization?: AssessmentSettingsData['allowMrqOptionsRandomization']; programming_max_time_limit: AssessmentSettingsData['maxProgrammingTimeLimit']; + rubric_grading_prompt_enabled?: AssessmentSettingsData['rubricGradingPromptEnabled']; + rubric_grading_prompt?: AssessmentSettingsData['rubricGradingPrompt']; assessment_categories_attributes?: { id: AssessmentCategory['id']; title: AssessmentCategory['title']; diff --git a/client/locales/en.json b/client/locales/en.json index aa30c20c106..eae07bb46bf 100644 --- a/client/locales/en.json +++ b/client/locales/en.json @@ -500,6 +500,15 @@ "course.admin.AssessmentSettings.programmingQuestionSettings": { "defaultMessage": "Programming Question settings" }, + "course.admin.AssessmentSettings.rubricGrading": { + "defaultMessage": "AI Rubric Grading" + }, + "course.admin.AssessmentSettings.rubricGradingPromptHint": { + "defaultMessage": "When a question with a grading rubric is autograded with AI, these instructions will be inserted before the question's specific grading prompt." + }, + "course.admin.AssessmentSettings.rubricGradingPromptRequired": { + "defaultMessage": "Please enter a grading prompt, or disable the course-wide grading prompt." + }, "course.admin.AssessmentSettings.seconds": { "defaultMessage": "s" }, @@ -515,6 +524,9 @@ "course.admin.AssessmentSettings.toTab": { "defaultMessage": "to {tab}" }, + "course.admin.AssessmentSettings.useRubricGradingPrompt": { + "defaultMessage": "Use course-wide grading prompt" + }, "course.admin.CodaveriSettings.codaveriModel": { "defaultMessage": "Model" }, diff --git a/client/locales/ko.json b/client/locales/ko.json index 10e11ce6b7b..67f76356040 100644 --- a/client/locales/ko.json +++ b/client/locales/ko.json @@ -500,6 +500,15 @@ "course.admin.AssessmentSettings.programmingQuestionSettings": { "defaultMessage": "프로그래밍 질문 설정" }, + "course.admin.AssessmentSettings.rubricGrading": { + "defaultMessage": "AI 루브릭 채점" + }, + "course.admin.AssessmentSettings.rubricGradingPromptHint": { + "defaultMessage": "채점 루브릭이 있는 문제를 AI로 자동 채점할 때, 이 지침이 해당 문제의 개별 채점 프롬프트 앞에 삽입됩니다." + }, + "course.admin.AssessmentSettings.rubricGradingPromptRequired": { + "defaultMessage": "채점 프롬프트를 입력하거나 코스 전체 채점 프롬프트를 비활성화하세요." + }, "course.admin.AssessmentSettings.seconds": { "defaultMessage": "초" }, @@ -515,6 +524,9 @@ "course.admin.AssessmentSettings.toTab": { "defaultMessage": "{tab}로" }, + "course.admin.AssessmentSettings.useRubricGradingPrompt": { + "defaultMessage": "코스 전체 채점 프롬프트 사용" + }, "course.admin.CodaveriSettings.Some": { "defaultMessage": "일부" }, diff --git a/client/locales/zh.json b/client/locales/zh.json index 7649ff1bab5..f6fd093c667 100644 --- a/client/locales/zh.json +++ b/client/locales/zh.json @@ -500,6 +500,15 @@ "course.admin.AssessmentSettings.programmingQuestionSettings": { "defaultMessage": "编程题设置" }, + "course.admin.AssessmentSettings.rubricGrading": { + "defaultMessage": "AI 评分标准评分" + }, + "course.admin.AssessmentSettings.rubricGradingPromptHint": { + "defaultMessage": "当带有评分标准的题目由 AI 自动评分时，这些说明将被插入到该题目专属评分提示之前。" + }, + "course.admin.AssessmentSettings.rubricGradingPromptRequired": { + "defaultMessage": "请输入评分提示，或停用课程级评分提示。" + }, "course.admin.AssessmentSettings.seconds": { "defaultMessage": "秒" }, @@ -515,6 +524,9 @@ "course.admin.AssessmentSettings.toTab": { "defaultMessage": "到 {tab}" }, + "course.admin.AssessmentSettings.useRubricGradingPrompt": { + "defaultMessage": "使用课程级评分提示" + }, "course.admin.CodaveriSettings.codaveriModel": { "defaultMessage": "模型" }, diff --git a/spec/controllers/course/admin/assessment_settings_controller_spec.rb b/spec/controllers/course/admin/assessment_settings_controller_spec.rb index 66a10223739..3f86eb9a48b 100644 --- a/spec/controllers/course/admin/assessment_settings_controller_spec.rb +++ b/spec/controllers/course/admin/assessment_settings_controller_spec.rb @@ -28,6 +28,21 @@ end end + describe '#update persisting the course-wide rubric grading prompt' do + subject do + patch :update, as: :json, params: { + course_id: course, + course: { rubric_grading_prompt_enabled: true, rubric_grading_prompt: 'Grade generously' } + } + end + + it 'stores the prompt and enable flag on the course settings' do + expect(subject).to render_template(:edit) + expect(course.reload.rubric_grading_prompt).to eq('Grade generously') + expect(course.reload.rubric_grading_prompt_enabled).to be(true) + end + end + describe 'moving actions' do let!(:category1) { create(:course_assessment_category, course: course) } let!(:category2) { create(:course_assessment_category, course: course) } diff --git a/spec/factories/course_assessment_answer_rubric_based_response.rb b/spec/factories/course_assessment_answer_rubric_based_response.rb index 72995018f26..ce9d5de0eac 100644 --- a/spec/factories/course_assessment_answer_rubric_based_response.rb +++ b/spec/factories/course_assessment_answer_rubric_based_response.rb @@ -12,17 +12,5 @@ assessment: assessment).question end answer_text { 'This is a sample response to the rubric question.' } - - trait :with_selections do - after(:build) do |answer, _evaluator| - question = answer.question.specific - question.categories.each do |category| - selection = build(:course_assessment_answer_rubric_based_response_selection, - answer: answer, - category: category) - answer.selections << selection - end - end - end end end diff --git a/spec/factories/course_assessment_answer_rubric_based_response_selection.rb b/spec/factories/course_assessment_answer_rubric_based_response_selection.rb deleted file mode 100644 index 3ccde5c3185..00000000000 --- a/spec/factories/course_assessment_answer_rubric_based_response_selection.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true -FactoryBot.define do - factory :course_assessment_answer_rubric_based_response_selection, - class: Course::Assessment::Answer::RubricBasedResponseSelection do - association :answer, factory: :course_assessment_answer_rubric_based_response - - transient do - question { answer.question.specific } - end - - category do - question.categories.first - end - - criterion { nil } - grade { nil } - explanation { nil } - end -end diff --git a/spec/models/course/assessment/question/forum_post_response_spec.rb b/spec/models/course/assessment/question/forum_post_response_spec.rb index d33239aeadf..6f3ea289ae9 100644 --- a/spec/models/course/assessment/question/forum_post_response_spec.rb +++ b/spec/models/course/assessment/question/forum_post_response_spec.rb @@ -59,6 +59,29 @@ end end + describe 'duplication of active_rubric' do + let(:course) { create(:course) } + let(:assessment) { create(:assessment, course: course) } + let(:question) do + create(:course_assessment_question_forum_post_response, assessment: assessment) + end + + before do + rubric = create(:course_rubric, course: course, questions: [question.acting_as]) + question.acting_as.update_columns(grading_mode: 'rubric', active_rubric_id: rubric.id) + end + + subject(:duplicate) do + Duplicator.new([], destination_course: course, current_course: course).duplicate(question) + end + + it 'gives the duplicate its own rubric of identical content rather than sharing the source rubric' do + expect(duplicate.active_rubric).to be_present + expect(duplicate.active_rubric).not_to eq(question.active_rubric) + expect(duplicate.active_rubric.canonical_content_hash).to eq(question.active_rubric.content_hash) + end + end + describe 'validations' do subject { build(:course_assessment_question_forum_post_response) } diff --git a/spec/models/course/assessment/question/grading_context_spec.rb b/spec/models/course/assessment/question/grading_context_spec.rb index 1d893223b65..4d31968aebf 100644 --- a/spec/models/course/assessment/question/grading_context_spec.rb +++ b/spec/models/course/assessment/question/grading_context_spec.rb @@ -70,31 +70,36 @@ def duplicate(objects) end end - context 'with a sibling_question_answer context' do - let(:source_question) do - create(:course_assessment_question_forum_post_response, assessment: assessment) - end - - before do - described_class.create!( - question: consumer.acting_as, context_type: 'sibling_question_answer', - source: source_question.acting_as, identifier: 'sibling' - ) - end + # A sibling source can be any question that provides_grading_context? (a rubric-graded forum question, or a + # plain text response that is only a source), so the linkage must survive duplication for both kinds. + { + 'a forum question' => :course_assessment_question_forum_post_response, + 'a text response' => :course_assessment_question_text_response + }.each do |source_desc, source_factory| + context "with a sibling_question_answer context sourced from #{source_desc}" do + let(:source_question) { create(source_factory, assessment: assessment) } + + before do + described_class.create!( + question: consumer.acting_as, context_type: 'sibling_question_answer', + source: source_question.acting_as, identifier: 'sibling' + ) + end - # The Duplicator imposes no order on the two co-duplicated questions, so the source linkage must hold - # whichever is processed first. - [%i[source_question consumer], %i[consumer source_question]].each do |order| - it "re-points the duplicate context at the duplicate source (order: #{order.join(' then ')})" do - originals = order.map { |name| send(name) } - dup_by_original = originals.zip(duplicate(originals)).to_h - dup_consumer = dup_by_original[consumer] - dup_source = dup_by_original[source_question] - - context = dup_consumer.grading_contexts.find { |c| c.context_type == 'sibling_question_answer' } - expect(context.identifier).to eq('sibling') - expect(context.source).to eq(dup_source.acting_as) - expect(context.source).not_to eq(source_question.acting_as) + # The Duplicator imposes no order on the two co-duplicated questions, so the source linkage must hold + # whichever is processed first. + [%i[source_question consumer], %i[consumer source_question]].each do |order| + it "re-points the duplicate context at the duplicate source (order: #{order.join(' then ')})" do + originals = order.map { |name| send(name) } + dup_by_original = originals.zip(duplicate(originals)).to_h + dup_consumer = dup_by_original[consumer] + dup_source = dup_by_original[source_question] + + context = dup_consumer.grading_contexts.find { |c| c.context_type == 'sibling_question_answer' } + expect(context.identifier).to eq('sibling') + expect(context.source).to eq(dup_source.acting_as) + expect(context.source).not_to eq(source_question.acting_as) + end end end end diff --git a/spec/models/course/rubric/rubric_adapter_spec.rb b/spec/models/course/rubric/rubric_adapter_spec.rb new file mode 100644 index 00000000000..c082c85d6bc --- /dev/null +++ b/spec/models/course/rubric/rubric_adapter_spec.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true +require 'rails_helper' + +RSpec.describe Course::Rubric::RubricAdapter do + let(:instance) { Instance.default } + with_tenant(:instance) do + let(:course) { create(:course) } + let(:rubric) { create(:course_rubric, course: course, grading_prompt: 'Question guidance') } + + subject(:grading_prompt) { described_class.new(rubric).grading_prompt } + + context 'when the course-wide grading prompt is disabled' do + before { course.update!(rubric_grading_prompt: 'Course-wide guidance', rubric_grading_prompt_enabled: false) } + + it 'returns just the question grading prompt' do + expect(grading_prompt).to eq('Question guidance') + end + end + + context 'when the course-wide grading prompt is enabled' do + before { course.update!(rubric_grading_prompt: 'Course-wide guidance', rubric_grading_prompt_enabled: true) } + + it 'prepends the course prompt before the question prompt' do + expect(grading_prompt).to eq("Course-wide guidance\n\nQuestion guidance") + end + + context 'and the question grading prompt is blank' do + let(:rubric) { create(:course_rubric, course: course, grading_prompt: '') } + + it 'returns just the course prompt, without a dangling separator' do + expect(grading_prompt).to eq('Course-wide guidance') + end + end + + context 'but the course prompt itself is blank' do + before { course.update!(rubric_grading_prompt: '') } + + it 'returns just the question grading prompt' do + expect(grading_prompt).to eq('Question guidance') + end + end + end + end +end